[ { "path": ".commitlintrc.json", "content": "{\n \"extends\": [\n \"@commitlint/config-conventional\"\n ]\n}\n" }, { "path": ".github/CODEOWNERS", "content": "* @neo4j/team-connectors\n\n/.github/ @ali-ince @fbiville @venikkin\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug report\nabout: Create a report to help us improve\nlabels: bug\n---\n\n## Guidelines\n\nPlease note that GitHub issues are only meant for bug reports/feature requests. If you have questions on how to use the Neo4j Connector for Apache Spark,\nplease ask on [the Neo4j Discussion Forum](https://community.neo4j.com/c/integrations/18) instead of creating an issue here.\n\n## Expected Behavior (Mandatory)\n\n\n## Actual Behavior (Mandatory)\n\n\n## How to Reproduce the Problem\n\n### Simple Dataset (where it's possible)\n\n```\n// Insert the output of the `df.show()` call\n\n```\n\n\n### Steps (Mandatory)\n\n1.\n1.\n1.\n\n## Screenshots (where it's possible)\n\n## Specifications (Mandatory)\n\nCurrently used versions\n\n### Versions\n\n- Spark:\n- Scala:\n- Neo4j:\n- Neo4j Connector:\n\n## Additional information\n\n* The code of the Spark job\n* the structure of the Dataframe\n* did you define the constraints/indexes?\n* if you're you using any Spark Cloud provider please specify it (ie: Databricks)\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature request\nabout: Suggest an idea for this project\n\n---\n\n## Guidelines\n\nPlease note that GitHub issues are only meant for bug reports/feature requests. If you have questions on how to use the Neo4j Connector for Apache Spark,\nplease ask on [the Neo4j Discussion Forum](https://community.neo4j.com/c/integrations/18) instead of creating an issue here.\n\n## Feature description (Mandatory)\nA clear and concise description of what you want to happen. Add any considered drawbacks.\n\n## Considered alternatives\nA clear and concise description of any alternative solutions or features you've considered. Maybe there is something in the project that could be reused?\n\n## How this feature can improve the project?\nIf you can, explain how users will be able to use this and possibly write out a version the docs.\nMaybe a screenshot or design?" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"daily\"\n cooldown:\n default-days: 3\n\n - package-ecosystem: \"maven\"\n directory: \"/\"\n target-branch: \"5.0\"\n schedule:\n interval: \"daily\"\n cooldown:\n default-days: 3\n\n - package-ecosystem: \"maven\"\n directory: \"/\"\n target-branch: \"6.0\"\n schedule:\n interval: \"daily\"\n cooldown:\n default-days: 3\n" }, { "path": ".gitignore", "content": "neo4j-home\n.gradle\ngradle/\nbuild/\n*~\n\\#*\ntarget\nout\n.project\n.classpath\n.settings\n.externalToolBuilders/\n.scala_dependencies\n.factorypath\n.cache\n.cache-main\n.cache-tests\n*.iws\n*.ipr\n*.iml\n.idea\n.DS_Store\n.shell_history\n.mailmap\n.java-version\n.cache-main\n.cache-tests\nThumbs.db\n.cache-main\n.cache-tests\ndocs/guides\ndoc/node\ndoc/node_modules\ndoc/package-lock.json\nscripts/python/local\nnode_modules\n" }, { "path": ".husky/commit-msg", "content": "#!/usr/bin/env sh\n\nnpx --no -- commitlint --edit \"$1\"\n" }, { "path": ".husky/pre-commit", "content": "#!/usr/bin/env sh\n\n./mvnw sortpom:sort spotless:apply -f .teamcity\n./mvnw sortpom:sort spotless:apply\ngit update-index --again\n\n" }, { "path": ".mvn/wrapper/maven-wrapper.properties", "content": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\nwrapperVersion=3.3.2\ndistributionType=only-script\ndistributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip\n" }, { "path": ".teamcity/.editorconfig", "content": "# This .editorconfig section approximates ktfmt's formatting rules. You can include it in an\n# existing .editorconfig file or use it standalone by copying it to /.editorconfig\n# and making sure your editor is set to read settings from .editorconfig files.\n#\n# It includes editor-specific config options for IntelliJ IDEA.\n#\n# If any option is wrong, PR are welcome\n\n[*]\nmax_line_length = unset\n\n[pom.xml]\nmax_line_length = 180\n\n[{*.kt,*.kts}]\nindent_style = space\ninsert_final_newline = true\nmax_line_length = 100\nindent_size = 2\nij_continuation_indent_size = 4\nij_java_names_count_to_use_import_on_demand = 9999\nij_kotlin_align_in_columns_case_branch = false\nij_kotlin_align_multiline_binary_operation = false\nij_kotlin_align_multiline_extends_list = false\nij_kotlin_align_multiline_method_parentheses = false\nij_kotlin_align_multiline_parameters = true\nij_kotlin_align_multiline_parameters_in_calls = false\nij_kotlin_allow_trailing_comma = true\nij_kotlin_allow_trailing_comma_on_call_site = true\nij_kotlin_assignment_wrap = normal\nij_kotlin_blank_lines_after_class_header = 0\nij_kotlin_blank_lines_around_block_when_branches = 0\nij_kotlin_blank_lines_before_declaration_with_comment_or_annotation_on_separate_line = 1\nij_kotlin_block_comment_at_first_column = true\nij_kotlin_call_parameters_new_line_after_left_paren = true\nij_kotlin_call_parameters_right_paren_on_new_line = false\nij_kotlin_call_parameters_wrap = on_every_item\nij_kotlin_catch_on_new_line = false\nij_kotlin_class_annotation_wrap = split_into_lines\nij_kotlin_code_style_defaults = KOTLIN_OFFICIAL\nij_kotlin_continuation_indent_for_chained_calls = true\nij_kotlin_continuation_indent_for_expression_bodies = true\nij_kotlin_continuation_indent_in_argument_lists = true\nij_kotlin_continuation_indent_in_elvis = false\nij_kotlin_continuation_indent_in_if_conditions = false\nij_kotlin_continuation_indent_in_parameter_lists = false\nij_kotlin_continuation_indent_in_supertype_lists = false\nij_kotlin_else_on_new_line = false\nij_kotlin_enum_constants_wrap = off\nij_kotlin_extends_list_wrap = normal\nij_kotlin_field_annotation_wrap = split_into_lines\nij_kotlin_finally_on_new_line = false\nij_kotlin_if_rparen_on_new_line = false\nij_kotlin_import_nested_classes = false\nij_kotlin_insert_whitespaces_in_simple_one_line_method = true\nij_kotlin_keep_blank_lines_before_right_brace = 2\nij_kotlin_keep_blank_lines_in_code = 2\nij_kotlin_keep_blank_lines_in_declarations = 2\nij_kotlin_keep_first_column_comment = true\nij_kotlin_keep_indents_on_empty_lines = false\nij_kotlin_keep_line_breaks = true\nij_kotlin_lbrace_on_next_line = false\nij_kotlin_line_comment_add_space = false\nij_kotlin_line_comment_at_first_column = true\nij_kotlin_method_annotation_wrap = split_into_lines\nij_kotlin_method_call_chain_wrap = normal\nij_kotlin_method_parameters_new_line_after_left_paren = true\nij_kotlin_method_parameters_right_paren_on_new_line = true\nij_kotlin_method_parameters_wrap = on_every_item\nij_kotlin_name_count_to_use_star_import = 9999\nij_kotlin_name_count_to_use_star_import_for_members = 9999\nij_kotlin_parameter_annotation_wrap = off\nij_kotlin_space_after_comma = true\nij_kotlin_space_after_extend_colon = true\nij_kotlin_space_after_type_colon = true\nij_kotlin_space_before_catch_parentheses = true\nij_kotlin_space_before_comma = false\nij_kotlin_space_before_extend_colon = true\nij_kotlin_space_before_for_parentheses = true\nij_kotlin_space_before_if_parentheses = true\nij_kotlin_space_before_lambda_arrow = true\nij_kotlin_space_before_type_colon = false\nij_kotlin_space_before_when_parentheses = true\nij_kotlin_space_before_while_parentheses = true\nij_kotlin_spaces_around_additive_operators = true\nij_kotlin_spaces_around_assignment_operators = true\nij_kotlin_spaces_around_equality_operators = true\nij_kotlin_spaces_around_function_type_arrow = true\nij_kotlin_spaces_around_logical_operators = true\nij_kotlin_spaces_around_multiplicative_operators = true\nij_kotlin_spaces_around_range = false\nij_kotlin_spaces_around_relational_operators = true\nij_kotlin_spaces_around_unary_operator = false\nij_kotlin_spaces_around_when_arrow = true\nij_kotlin_variable_annotation_wrap = off\nij_kotlin_while_on_new_line = false\nij_kotlin_wrap_elvis_expressions = 1\nij_kotlin_wrap_expression_body_functions = 1\nij_kotlin_wrap_first_method_in_call_chain = false" }, { "path": ".teamcity/builds/Build.kt", "content": "package builds\n\nimport jetbrains.buildServer.configs.kotlin.BuildType\nimport jetbrains.buildServer.configs.kotlin.Project\nimport jetbrains.buildServer.configs.kotlin.buildFeatures.notifications\nimport jetbrains.buildServer.configs.kotlin.sequential\nimport jetbrains.buildServer.configs.kotlin.toId\n\nclass Build(\n name: String,\n forPullRequests: Boolean,\n javaVersions: Set,\n scalaVersions: Set,\n pysparkVersions: Set,\n neo4jVersions: Set,\n forCompatibility: Boolean = false,\n customizeCompletion: BuildType.() -> Unit = {}\n) :\n Project(\n {\n this.id(name.toId())\n this.name = name\n\n val complete = Empty(\"${name}-complete\", \"complete\")\n\n val bts = sequential {\n if (forPullRequests)\n buildType(WhiteListCheck(\"${name}-whitelist-check\", \"white-list check\"))\n if (forPullRequests) dependentBuildType(PRCheck(\"${name}-pr-check\", \"pr check\"))\n\n parallel {\n scalaVersions.forEach { scala ->\n dependentBuildType(\n SemgrepCheck(\n \"${name}-semgrep-check-${scala.version}\",\n \"semgrep check (${scala.version})\",\n scala))\n }\n\n javaVersions.cartesianProduct(scalaVersions, neo4jVersions).forEach {\n (java, scala, neo4j) ->\n sequential {\n val packaging =\n Package(\n \"${name}-package-${java.version}-${scala.version}-${neo4j.version}\",\n \"package (${java.version}, ${scala.version}, ${neo4j.version})\",\n java,\n scala,\n )\n\n dependentBuildType(\n Maven(\n \"${name}-build-${java.version}-${scala.version}-${neo4j.version}\",\n \"build (${java.version}, ${scala.version}, ${neo4j.version})\",\n \"test-compile\",\n java,\n scala,\n ),\n )\n\n dependentBuildType(\n Maven(\n \"${name}-unit-tests-${java.version}-${scala.version}-${neo4j.version}\",\n \"unit tests (${java.version}, ${scala.version}, ${neo4j.version})\",\n \"test\",\n java,\n scala,\n neo4j,\n ),\n )\n\n dependentBuildType(\n collectArtifacts(\n packaging,\n ),\n )\n\n parallel {\n dependentBuildType(\n JavaIntegrationTests(\n \"${name}-integration-tests-java-${java.version}-${scala.version}-${neo4j.version}\",\n \"java integration tests (${java.version}, ${scala.version}, ${neo4j.version})\",\n java,\n scala,\n neo4j,\n ) {},\n )\n\n pysparkVersions\n .filter { it.shouldTestWith(java, scala) }\n .forEach { pyspark ->\n pyspark.pythonVersions.forEach { python ->\n dependentBuildType(\n PythonIntegrationTests(\n \"${name}-integration-tests-pyspark-${java.version}-${scala.version}-${neo4j.version}-${python.version}-${pyspark.sparkVersion.version}\",\n \"pyspark integration tests (${java.version}, ${scala.version}, ${neo4j.version}, ${python.version}, ${pyspark.sparkVersion.version})\",\n java,\n python,\n scala,\n pyspark.sparkVersion,\n neo4j,\n ) {\n dependencies {\n artifacts(packaging) {\n artifactRules =\n \"\"\"\n +:packages/*.jar => ./scripts/python\n \"\"\"\n .trimIndent()\n }\n }\n },\n )\n }\n }\n }\n }\n }\n }\n\n dependentBuildType(complete)\n if (!forPullRequests && !forCompatibility)\n dependentBuildType(Release(\"${name}-release\", \"release\", DEFAULT_JAVA_VERSION))\n }\n\n bts.buildTypes().forEach {\n it.thisVcs(if (forPullRequests) \"pull/*\" else DEFAULT_BRANCH)\n\n it.features {\n loginToECR()\n requireDiskSpace(\"5gb\")\n if (!forCompatibility) enableCommitStatusPublisher()\n if (forPullRequests) enablePullRequests()\n }\n\n buildType(it)\n }\n\n complete.features {\n notifications {\n branchFilter = buildString {\n appendLine(\"+:$DEFAULT_BRANCH\")\n appendLine(\"+:refs/heads/$DEFAULT_BRANCH\")\n if (forPullRequests) {\n appendLine(\"+:pull/*\")\n appendLine(\"+:refs/heads/pull/*\")\n }\n }\n\n queuedBuildRequiresApproval = forPullRequests\n buildFailedToStart = !forPullRequests\n buildFailed = !forPullRequests\n buildFinishedSuccessfully = !forPullRequests\n buildProbablyHanging = !forPullRequests\n\n notifierSettings = slackNotifier {\n connection = SLACK_CONNECTION_ID\n sendTo = SLACK_CHANNEL\n messageFormat = simpleMessageFormat()\n }\n }\n }\n\n complete.apply(customizeCompletion)\n },\n )\n" }, { "path": ".teamcity/builds/Common.kt", "content": "package builds\n\nimport builds.Neo4jSparkConnectorVcs.branchSpec\nimport jetbrains.buildServer.configs.kotlin.BuildFeatures\nimport jetbrains.buildServer.configs.kotlin.BuildSteps\nimport jetbrains.buildServer.configs.kotlin.BuildType\nimport jetbrains.buildServer.configs.kotlin.CompoundStage\nimport jetbrains.buildServer.configs.kotlin.FailureAction\nimport jetbrains.buildServer.configs.kotlin.Requirements\nimport jetbrains.buildServer.configs.kotlin.ReuseBuilds\nimport jetbrains.buildServer.configs.kotlin.buildFeatures.PullRequests\nimport jetbrains.buildServer.configs.kotlin.buildFeatures.buildCache\nimport jetbrains.buildServer.configs.kotlin.buildFeatures.commitStatusPublisher\nimport jetbrains.buildServer.configs.kotlin.buildFeatures.dockerRegistryConnections\nimport jetbrains.buildServer.configs.kotlin.buildFeatures.freeDiskSpace\nimport jetbrains.buildServer.configs.kotlin.buildFeatures.pullRequests\nimport jetbrains.buildServer.configs.kotlin.buildSteps.DockerCommandStep\nimport jetbrains.buildServer.configs.kotlin.buildSteps.MavenBuildStep\nimport jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep\nimport jetbrains.buildServer.configs.kotlin.buildSteps.dockerCommand\nimport jetbrains.buildServer.configs.kotlin.buildSteps.maven\nimport jetbrains.buildServer.configs.kotlin.buildSteps.script\nimport jetbrains.buildServer.configs.kotlin.vcs.GitVcsRoot\n\nconst val GITHUB_OWNER = \"neo4j\"\nconst val GITHUB_REPOSITORY = \"neo4j-spark-connector\"\nconst val DEFAULT_BRANCH = \"5.0\"\nval MAVEN_DEFAULT_ARGS = buildString {\n append(\"--no-transfer-progress \")\n append(\"--batch-mode \")\n append(\"-Dmaven.repo.local=%teamcity.build.checkoutDir%/.m2/repository \")\n append(\"-Dmaven.wagon.http.retryHandler.class=standard \")\n append(\"-Dmaven.wagon.http.retryHandler.timeout=60 \")\n append(\"-Dmaven.wagon.http.retryHandler.count=3 \")\n append(\n \"-Dmaven.wagon.http.retryHandler.nonRetryableClasses=java.io.InterruptedIOException,java.net.UnknownHostException,java.net.ConnectException \")\n}\nconst val SEMGREP_DOCKER_IMAGE = \"semgrep/semgrep:1.146.0\"\nconst val FULL_GITHUB_REPOSITORY = \"$GITHUB_OWNER/$GITHUB_REPOSITORY\"\nconst val GITHUB_URL = \"https://github.com/$FULL_GITHUB_REPOSITORY\"\n\nval DEFAULT_JAVA_VERSION = JavaVersion.V_11\n\n// Look into Root Project's settings -> Connections\nconst val SLACK_CONNECTION_ID = \"PROJECT_EXT_83\"\nconst val SLACK_CHANNEL = \"#team-connectors-feed\"\n\n// Look into Root Project's settings -> Connections\nconst val ECR_CONNECTION_ID = \"PROJECT_EXT_124\"\n\nenum class LinuxSize(val value: String) {\n SMALL(\"small\"),\n LARGE(\"large\")\n}\n\nenum class JavaVersion(val version: String, val dockerImage: String) {\n V_8(version = \"8\", dockerImage = \"eclipse-temurin:8-jdk\"),\n V_11(version = \"11\", dockerImage = \"eclipse-temurin:11-jdk\"),\n V_17(version = \"17\", dockerImage = \"eclipse-temurin:17-jdk\"),\n V_21(version = \"21\", dockerImage = \"eclipse-temurin:21-jdk\"),\n}\n\nenum class ScalaVersion(val version: String) {\n V2_12(version = \"2.12\"),\n V2_13(version = \"2.13\"),\n}\n\nenum class PythonVersion(val version: String) {\n V3_9(version = \"3.9\"),\n V3_10(version = \"3.10\"),\n V3_11(version = \"3.11\"),\n V3_12(version = \"3.12\"),\n V3_13(version = \"3.13\"),\n}\n\nenum class SparkVersion(val short: String, val version: String) {\n V3_4_4(short = \"3\", version = \"3.4.4\"),\n V3_5_5(short = \"3\", version = \"3.5.5\"),\n}\n\nenum class PySparkVersion(\n val sparkVersion: SparkVersion,\n val scalaVersion: ScalaVersion,\n val javaVersions: Set,\n val pythonVersions: Set,\n) {\n V3_4(\n SparkVersion.V3_4_4,\n ScalaVersion.V2_12,\n setOf(\n JavaVersion.V_8,\n JavaVersion.V_11,\n JavaVersion.V_17,\n ),\n setOf(\n PythonVersion.V3_9,\n PythonVersion.V3_10,\n PythonVersion.V3_11,\n PythonVersion.V3_12,\n ),\n ),\n V3_5(\n SparkVersion.V3_5_5,\n ScalaVersion.V2_12,\n setOf(\n JavaVersion.V_8,\n JavaVersion.V_11,\n JavaVersion.V_17,\n JavaVersion.V_21,\n ),\n setOf(\n PythonVersion.V3_9,\n PythonVersion.V3_10,\n PythonVersion.V3_11,\n PythonVersion.V3_12,\n PythonVersion.V3_13,\n ),\n ),\n}\n\nfun PySparkVersion.shouldTestWith(javaVersion: JavaVersion, scalaVersion: ScalaVersion): Boolean =\n this.javaVersions.contains(javaVersion) && this.scalaVersion == scalaVersion\n\nenum class Neo4jVersion(val version: String, val dockerImage: String) {\n V_NONE(\"\", \"\"),\n V_4_4(\"4.4\", \"neo4j:4.4-enterprise\"),\n V_4_4_DEV(\n \"4.4-dev\",\n \"535893049302.dkr.ecr.eu-west-1.amazonaws.com/build-service/neo4j:4.4-enterprise-debian-nightly\",\n ),\n V_5(\"5\", \"neo4j:5-enterprise\"),\n V_5_DEV(\n \"5-dev\",\n \"535893049302.dkr.ecr.eu-west-1.amazonaws.com/build-service/neo4j:5-enterprise-debian-nightly-bundle\",\n ),\n V_CALVER(\"2026\", \"neo4j:2026-enterprise\"),\n V_CALVER_DEV(\n \"2026-dev\",\n \"535893049302.dkr.ecr.eu-west-1.amazonaws.com/build-service/neo4j:2026-enterprise-debian-nightly-bundle\",\n ),\n}\n\nfun Iterable.cartesianProduct(\n other1: Collection,\n other2: Collection\n): Iterable> =\n this.flatMap { s -> other1.map { t -> s to t } }\n .flatMap { (s, t) -> other2.map { y -> Triple(s, t, y) } }\n\nobject Neo4jSparkConnectorVcs :\n GitVcsRoot(\n {\n id(\"Connectors_Neo4jSparkConnector_Build\")\n\n name = \"neo4j-spark-connector\"\n url = \"git@github.com:neo4j/neo4j-spark-connector.git\"\n branch = \"refs/heads/$DEFAULT_BRANCH\"\n branchSpec = \"refs/heads/*\"\n\n authMethod = defaultPrivateKey { userName = \"git\" }\n },\n )\n\nfun Requirements.runOnLinux(size: LinuxSize = LinuxSize.SMALL) {\n startsWith(\"cloud.amazon.agent-name-prefix\", \"linux-${size.value}\")\n}\n\nfun BuildType.thisVcs(forBranch: String) = vcs {\n root(Neo4jSparkConnectorVcs)\n\n branchSpec = buildString {\n appendLine(\"-:*\")\n appendLine(\"+:$forBranch\")\n }\n\n cleanCheckout = true\n}\n\nfun BuildFeatures.enableCommitStatusPublisher() = commitStatusPublisher {\n vcsRootExtId = Neo4jSparkConnectorVcs.id.toString()\n publisher = github {\n githubUrl = \"https://api.github.com\"\n authType = personalToken { token = \"%github-commit-status-token%\" }\n }\n}\n\nfun BuildFeatures.enablePullRequests() = pullRequests {\n vcsRootExtId = Neo4jSparkConnectorVcs.id.toString()\n provider = github {\n authType = token { token = \"%github-pull-request-token%\" }\n filterAuthorRole = PullRequests.GitHubRoleFilter.EVERYBODY\n filterTargetBranch = buildString {\n appendLine(\"+:$DEFAULT_BRANCH\")\n appendLine(\"+:refs/heads/$DEFAULT_BRANCH\")\n }\n }\n}\n\nfun BuildFeatures.requireDiskSpace(size: String = \"3gb\") = freeDiskSpace {\n requiredSpace = size\n failBuild = true\n}\n\nfun BuildFeatures.loginToECR() = dockerRegistryConnections {\n cleanupPushedImages = true\n loginToRegistry = on { dockerRegistryId = ECR_CONNECTION_ID }\n}\n\nfun BuildFeatures.buildCache(javaVersion: JavaVersion, scalaVersion: ScalaVersion) = buildCache {\n this.name =\n \"neo4j-spark-connector-${DEFAULT_BRANCH}-${javaVersion.version}-${scalaVersion.version}\"\n publish = true\n use = true\n publishOnlyChanged = true\n rules = \".m2/repository\"\n}\n\nfun CompoundStage.dependentBuildType(bt: BuildType, reuse: ReuseBuilds = ReuseBuilds.SUCCESSFUL) =\n buildType(bt) {\n onDependencyCancel = FailureAction.CANCEL\n onDependencyFailure = FailureAction.FAIL_TO_START\n reuseBuilds = reuse\n }\n\nfun collectArtifacts(buildType: BuildType): BuildType {\n buildType.artifactRules =\n \"\"\"\n +:spark-3/target/*_for_spark_*.jar => packages\n +:spark-3/target/*.zip => packages\n \"\"\"\n .trimIndent()\n\n return buildType\n}\n\nfun BuildSteps.runMaven(javaVersion: JavaVersion, init: MavenBuildStep.() -> Unit): MavenBuildStep {\n val maven =\n this.maven {\n dockerImagePlatform = MavenBuildStep.ImagePlatform.Linux\n dockerImage = javaVersion.dockerImage\n dockerRunParameters = \"--volume /var/run/docker.sock:/var/run/docker.sock\"\n\n localRepoScope = MavenBuildStep.RepositoryScope.MAVEN_DEFAULT\n }\n\n init(maven)\n return maven\n}\n\nfun BuildSteps.setVersion(name: String, version: String, javaVersion: JavaVersion): MavenBuildStep {\n return this.runMaven(javaVersion) {\n this.name = name\n goals = \"versions:set\"\n runnerArgs =\n \"$MAVEN_DEFAULT_ARGS -Djava.version=${javaVersion.version} -DnewVersion=$version -DgenerateBackupPoms=false\"\n }\n}\n\nfun BuildSteps.commitAndPush(\n name: String,\n commitMessage: String,\n includeFiles: String = \"\\\\*pom.xml\",\n dryRunParameter: String = \"dry-run\"\n): ScriptBuildStep {\n return this.script {\n this.name = name\n scriptContent =\n \"\"\"\n #!/bin/bash -eu \n \n git add $includeFiles\n git commit -m \"$commitMessage\"\n git push\n \"\"\"\n .trimIndent()\n\n conditions { doesNotMatch(dryRunParameter, \"true\") }\n }\n}\n\nfun BuildSteps.pullImage(version: Neo4jVersion): DockerCommandStep =\n this.dockerCommand {\n name = \"pull neo4j test image\"\n commandType = other {\n subCommand = \"image\"\n commandArgs = \"pull ${version.dockerImage}\"\n }\n }\n" }, { "path": ".teamcity/builds/Empty.kt", "content": "package builds\n\nimport jetbrains.buildServer.configs.kotlin.BuildType\nimport jetbrains.buildServer.configs.kotlin.toId\n\nclass Empty(id: String, name: String) :\n BuildType({\n this.id(id.toId())\n this.name = name\n\n requirements { runOnLinux(LinuxSize.SMALL) }\n })\n" }, { "path": ".teamcity/builds/JavaIntegrationTests.kt", "content": "package builds\n\nimport jetbrains.buildServer.configs.kotlin.BuildType\nimport jetbrains.buildServer.configs.kotlin.toId\n\nclass JavaIntegrationTests(\n id: String,\n name: String,\n javaVersion: JavaVersion,\n scalaVersion: ScalaVersion,\n neo4jVersion: Neo4jVersion,\n init: BuildType.() -> Unit\n) :\n BuildType(\n {\n this.id(id.toId())\n this.name = name\n\n init()\n\n artifactRules =\n \"\"\"\n +:diagnostics => diagnostics.zip\n \"\"\"\n .trimIndent()\n\n params { text(\"env.NEO4J_TEST_IMAGE\", neo4jVersion.dockerImage) }\n\n steps {\n if (neo4jVersion != Neo4jVersion.V_NONE) {\n pullImage(neo4jVersion)\n }\n\n runMaven(javaVersion) {\n this.goals = \"verify\"\n this.runnerArgs =\n \"$MAVEN_DEFAULT_ARGS -Djava.version=${javaVersion.version} -Dscala-${scalaVersion.version} -DskipUnitTests\"\n }\n }\n\n features { buildCache(javaVersion, scalaVersion) }\n\n requirements { runOnLinux(LinuxSize.LARGE) }\n },\n )\n" }, { "path": ".teamcity/builds/Maven.kt", "content": "package builds\n\nimport jetbrains.buildServer.configs.kotlin.BuildType\nimport jetbrains.buildServer.configs.kotlin.toId\n\nopen class Maven(\n id: String,\n name: String,\n goals: String,\n javaVersion: JavaVersion,\n scalaVersion: ScalaVersion,\n neo4jVersion: Neo4jVersion = Neo4jVersion.V_NONE,\n args: String? = null\n) :\n BuildType(\n {\n this.id(id.toId())\n this.name = name\n\n params {\n text(\"env.JAVA_VERSION\", javaVersion.version)\n text(\"env.NEO4J_TEST_IMAGE\", neo4jVersion.dockerImage)\n }\n\n steps {\n if (neo4jVersion != Neo4jVersion.V_NONE) {\n pullImage(neo4jVersion)\n }\n\n runMaven(javaVersion) {\n this.goals = goals\n this.runnerArgs =\n \"$MAVEN_DEFAULT_ARGS -Djava.version=${javaVersion.version} -Dscala-${scalaVersion.version} ${args ?: \"\"}\"\n }\n }\n\n features { buildCache(javaVersion, scalaVersion) }\n\n requirements { runOnLinux(LinuxSize.SMALL) }\n },\n )\n" }, { "path": ".teamcity/builds/PRCheck.kt", "content": "package builds\n\nimport jetbrains.buildServer.configs.kotlin.BuildType\nimport jetbrains.buildServer.configs.kotlin.buildFeatures.dockerSupport\nimport jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep\nimport jetbrains.buildServer.configs.kotlin.buildSteps.script\nimport jetbrains.buildServer.configs.kotlin.toId\n\nclass PRCheck(id: String, name: String) :\n BuildType({\n this.id(id.toId())\n this.name = name\n\n steps {\n script {\n scriptContent =\n \"\"\"\n #!/bin/bash\n \n set -eu\n \n export DANGER_GITHUB_API_TOKEN=%github-pull-request-token%\n export PULL_REQUEST_URL=https://github.com/$GITHUB_OWNER/$GITHUB_REPOSITORY/%teamcity.build.branch%\n \n # process pull request\n npm ci\n npx danger ci --verbose --failOnErrors\n \"\"\"\n .trimIndent()\n\n dockerImage = \"node:18.4\"\n dockerImagePlatform = ScriptBuildStep.ImagePlatform.Linux\n }\n }\n\n features { dockerSupport {} }\n\n requirements { runOnLinux(LinuxSize.SMALL) }\n })\n" }, { "path": ".teamcity/builds/Package.kt", "content": "package builds\n\nimport jetbrains.buildServer.configs.kotlin.BuildType\nimport jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep\nimport jetbrains.buildServer.configs.kotlin.buildSteps.script\nimport jetbrains.buildServer.configs.kotlin.toId\n\nclass Package(\n id: String,\n name: String,\n javaVersion: JavaVersion,\n scalaVersion: ScalaVersion,\n) :\n BuildType({\n this.id(id.toId())\n this.name = name\n\n params { text(\"env.JAVA_VERSION\", javaVersion.version) }\n\n steps {\n script {\n scriptContent =\n \"\"\"\n ./maven-release.sh package ${scalaVersion.version}\n \"\"\"\n .trimIndent()\n\n dockerImagePlatform = ScriptBuildStep.ImagePlatform.Linux\n dockerImage = javaVersion.dockerImage\n dockerRunParameters = \"--volume /var/run/docker.sock:/var/run/docker.sock\"\n }\n }\n\n features { buildCache(javaVersion, scalaVersion) }\n\n requirements { runOnLinux(LinuxSize.SMALL) }\n })\n" }, { "path": ".teamcity/builds/PythonIntegrationTests.kt", "content": "package builds\n\nimport jetbrains.buildServer.configs.kotlin.BuildType\nimport jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep\nimport jetbrains.buildServer.configs.kotlin.buildSteps.script\nimport jetbrains.buildServer.configs.kotlin.toId\n\nclass PythonIntegrationTests(\n id: String,\n name: String,\n javaVersion: JavaVersion,\n pythonVersion: PythonVersion,\n scalaVersion: ScalaVersion,\n sparkVersion: SparkVersion,\n neo4jVersion: Neo4jVersion,\n init: BuildType.() -> Unit\n) :\n BuildType(\n {\n this.id(id.toId())\n this.name = name\n\n init()\n\n artifactRules =\n \"\"\"\n +:diagnostics => diagnostics.zip\n \"\"\"\n .trimIndent()\n\n params { text(\"env.NEO4J_TEST_IMAGE\", neo4jVersion.dockerImage) }\n\n steps {\n if (neo4jVersion != Neo4jVersion.V_NONE) {\n pullImage(neo4jVersion)\n }\n\n script {\n scriptContent =\n \"\"\"\n #!/bin/bash -eu\n \n apt-get update\n apt-get install -o Acquire::Retries=10 --yes build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl git libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev\n curl -fsSL https://pyenv.run | bash\n \n export PYENV_ROOT=\"${'$'}HOME/.pyenv\"\n export PATH=\"${'$'}PYENV_ROOT/bin:${'$'}PATH\"\n eval \"$(pyenv init - bash)\"\n pyenv install ${pythonVersion.version}\n pyenv global ${pythonVersion.version}\n \n python -m pip install --upgrade pip\n pip install pyspark==${sparkVersion.version} \"testcontainers[neo4j]\" six tzlocal==2.1 \n \n project_version=\"$(./mvnw help:evaluate -Dexpression=\"project.version\" --quiet -DforceStdout)\"\n jar_name=\"neo4j-connector-apache-spark_${scalaVersion.version}-${'$'}{project_version}_for_spark_${sparkVersion.short}.jar\"\n cd ./scripts/python\n python test_spark.py \"${'$'}{jar_name}\" \"${neo4jVersion.dockerImage}\"\n \"\"\"\n .trimIndent()\n\n dockerImagePlatform = ScriptBuildStep.ImagePlatform.Linux\n dockerImage = javaVersion.dockerImage\n dockerRunParameters = \"--volume /var/run/docker.sock:/var/run/docker.sock\"\n }\n }\n\n requirements { runOnLinux(LinuxSize.SMALL) }\n },\n )\n" }, { "path": ".teamcity/builds/Release.kt", "content": "package builds\n\nimport jetbrains.buildServer.configs.kotlin.AbsoluteId\nimport jetbrains.buildServer.configs.kotlin.BuildType\nimport jetbrains.buildServer.configs.kotlin.ParameterDisplay\nimport jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep\nimport jetbrains.buildServer.configs.kotlin.buildSteps.script\nimport jetbrains.buildServer.configs.kotlin.toId\n\nprivate const val DRY_RUN = \"dry-run\"\n\nclass Release(id: String, name: String, javaVersion: JavaVersion) :\n BuildType(\n {\n this.id(id.toId())\n this.name = name\n\n templates(AbsoluteId(\"FetchSigningKey\"))\n\n params {\n text(\n \"releaseVersion\",\n \"\",\n label = \"Version to release\",\n display = ParameterDisplay.PROMPT,\n allowEmpty = false,\n )\n text(\n \"nextSnapshotVersion\",\n \"\",\n label = \"Next snapshot version\",\n description = \"Next snapshot version to set after release\",\n display = ParameterDisplay.PROMPT,\n allowEmpty = false,\n )\n\n checkbox(\n DRY_RUN,\n \"true\",\n \"Dry run?\",\n description =\n \"Whether to perform a dry run where nothing is published and released\",\n display = ParameterDisplay.PROMPT,\n checked = \"true\",\n unchecked = \"false\",\n )\n\n password(\"env.JRELEASER_GITHUB_TOKEN\", \"%github-pull-request-token%\")\n\n text(\"env.JRELEASER_DRY_RUN\", \"%$DRY_RUN%\")\n text(\"env.JRELEASER_PROJECT_VERSION\", \"%releaseVersion%\")\n\n text(\"env.JRELEASER_ANNOUNCE_SLACK_ACTIVE\", \"NEVER\")\n text(\"env.JRELEASER_ANNOUNCE_SLACK_TOKEN\", \"%slack-token%\")\n text(\"env.JRELEASER_ANNOUNCE_SLACK_WEBHOOK\", \"%slack-webhook%\")\n\n password(\"env.JRELEASER_GPG_PASSPHRASE\", \"%signing-key-passphrase%\")\n\n text(\"env.JRELEASER_MAVENCENTRAL_USERNAME\", \"%publish-username%\")\n password(\"env.JRELEASER_MAVENCENTRAL_TOKEN\", \"%publish-password%\")\n }\n\n steps {\n setVersion(\"Set release version\", \"%releaseVersion%\", javaVersion)\n\n commitAndPush(\n \"Push release version\",\n \"build: release version %releaseVersion%\",\n dryRunParameter = DRY_RUN,\n )\n\n script {\n scriptContent =\n \"\"\"\n #!/bin/bash\n \n set -eux\n \n apt-get update\n apt-get install -o Acquire::Retries=10 --yes build-essential curl git unzip zip\n \n # Get the jreleaser downloader\n curl -sL https://raw.githubusercontent.com/jreleaser/release-action/refs/tags/2.5.0/get_jreleaser.java > get_jreleaser.java\n\n # Download JReleaser\n java get_jreleaser.java 1.22.0\n\n if [ \"%dry-run%\" = \"true\" ]; then\n echo \"we are on a dry run, only performing upload to maven central\"\n export JRELEASER_MAVENCENTRAL_STAGE=UPLOAD\n export JRELEASER_ANNOUNCE_SLACK_ACTIVE=NEVER\n else\n echo \"we will do a full deploy to maven central\"\n export JRELEASER_MAVENCENTRAL_STAGE=FULL\n export JRELEASER_ANNOUNCE_SLACK_ACTIVE=ALWAYS\n fi\n \n # Execute JReleaser\n java -jar jreleaser-cli.jar assemble\n java -jar jreleaser-cli.jar full-release --debug\n \"\"\"\n .trimIndent()\n\n dockerImagePlatform = ScriptBuildStep.ImagePlatform.Linux\n dockerImage = javaVersion.dockerImage\n dockerRunParameters =\n \"--volume /var/run/docker.sock:/var/run/docker.sock --volume %teamcity.build.checkoutDir%/signingkeysandbox:/root/.gnupg\"\n }\n\n setVersion(\"Set next snapshot version\", \"%nextSnapshotVersion%\", javaVersion)\n\n commitAndPush(\n \"Push next snapshot version\",\n \"build: update version to %nextSnapshotVersion%\",\n dryRunParameter = DRY_RUN,\n )\n }\n\n artifactRules =\n \"\"\"\n +:artifacts => artifacts\n +:out/jreleaser => jreleaser\n \"\"\"\n .trimIndent()\n\n dependencies {\n artifacts(AbsoluteId(\"Tools_ReleaseTool\")) {\n buildRule = lastSuccessful()\n artifactRules = \"rt.jar => lib\"\n }\n }\n\n requirements { runOnLinux(LinuxSize.SMALL) }\n },\n )\n" }, { "path": ".teamcity/builds/SemgrepCheck.kt", "content": "package builds\n\nimport jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep\n\nclass SemgrepCheck(id: String, name: String, scalaVersion: ScalaVersion) :\n Maven(\n id,\n name,\n \"dependency:tree\",\n JavaVersion.V_17,\n scalaVersion,\n Neo4jVersion.V_NONE,\n \"-DoutputFile=maven_dep_tree.txt\") {\n\n init {\n\n params.password(\"env.SEMGREP_APP_TOKEN\", \"%semgrep-app-token%\")\n params.text(\"env.SEMGREP_REPO_NAME\", FULL_GITHUB_REPOSITORY)\n params.text(\"env.SEMGREP_REPO_URL\", GITHUB_URL)\n params.text(\"env.SEMGREP_BRANCH\", \"%teamcity.build.branch%\")\n params.text(\"env.SEMGREP_JOB_URL\", \"%env.BUILD_URL%\")\n params.text(\"env.SEMGREP_COMMIT\", \"%env.BUILD_VCS_NUMBER%\")\n\n steps.step(\n ScriptBuildStep {\n scriptContent = \"semgrep ci --no-git-ignore\"\n dockerImagePlatform = ScriptBuildStep.ImagePlatform.Linux\n dockerImage = SEMGREP_DOCKER_IMAGE\n dockerRunParameters =\n \"--volume /var/run/docker.sock:/var/run/docker.sock --volume %teamcity.build.checkoutDir%/signingkeysandbox:/root/.gnupg\"\n })\n }\n}\n" }, { "path": ".teamcity/builds/WhiteListCheck.kt", "content": "package builds\n\nimport jetbrains.buildServer.configs.kotlin.AbsoluteId\nimport jetbrains.buildServer.configs.kotlin.BuildType\nimport jetbrains.buildServer.configs.kotlin.buildSteps.script\nimport jetbrains.buildServer.configs.kotlin.toId\n\nclass WhiteListCheck(id: String, name: String) :\n BuildType({\n this.id(id.toId())\n this.name = name\n\n dependencies {\n artifacts(AbsoluteId(\"Tools_WhitelistCheck\")) {\n buildRule = lastSuccessful()\n cleanDestination = true\n artifactRules = \"whitelist-check.tar.gz!** => whitelist-check/\"\n }\n }\n\n steps {\n script {\n scriptContent =\n \"\"\"\n #!/bin/bash -eu\n \n BRANCH=%teamcity.pullRequest.source.branch%\n if [[ \"${'$'}BRANCH\" =~ dependabot/.* ]]; then\n echo \"Raised by dependabot, skipping the white list check\"\n exit 0\n fi\n \n echo \"Checking committers on PR %teamcity.build.branch%\"\n TOKEN=\"%github-pull-request-token%\"\n \n # process pull request\n ./whitelist-check/bin/examine-pull-request $GITHUB_OWNER $GITHUB_REPOSITORY \"${'$'}{TOKEN}\" %teamcity.build.branch% whitelist-check/cla-database.csv\n \"\"\"\n .trimIndent()\n formatStderrAsError = true\n }\n }\n\n requirements { runOnLinux(LinuxSize.SMALL) }\n })\n" }, { "path": ".teamcity/pom.xml", "content": "\n\n 4.0.0\n \n org.jetbrains.teamcity\n configs-dsl-kotlin-parent\n 1.0-SNAPSHOT\n \n \n Connectors_Neo4jSparkConnector\n teamcity-pipeline\n 1.0-SNAPSHOT\n Connectors_Neo4jSparkConnector Config DSL Script\n \n UTF-8\n 4.0.0\n 2.40.0\n \n \n \n org.jetbrains.kotlin\n kotlin-script-runtime\n ${kotlin.version}\n compile\n \n \n org.jetbrains.kotlin\n kotlin-stdlib-jdk8\n ${kotlin.version}\n compile\n \n \n org.jetbrains.teamcity\n configs-dsl-kotlin-latest\n ${teamcity.dsl.version}\n compile\n \n \n org.jetbrains.teamcity\n configs-dsl-kotlin-plugins-latest\n 1.0-SNAPSHOT\n pom\n compile\n \n \n \n \n \n true\n \n jetbrains-all\n https://download.jetbrains.com/teamcity-repository\n \n \n \n true\n \n teamcity-server\n https://live.neo4j-build.io/app/dsl-plugins-repository\n \n \n \n \n JetBrains\n https://download.jetbrains.com/teamcity-repository\n \n \n \n \n \n org.jetbrains.kotlin\n kotlin-maven-plugin\n ${kotlin.version}\n \n \n \n compile\n \n compile\n \n process-sources\n \n \n test-compile\n \n test-compile\n \n process-test-sources\n \n \n \n \n org.jetbrains.teamcity\n teamcity-configs-maven-plugin\n ${teamcity.dsl.version}\n \n kotlin\n target/generated-configs\n \n \n \n com.github.ekryd.sortpom\n sortpom-maven-plugin\n ${sortpom-maven-plugin.version}\n \n ${project.build.sourceEncoding}\n false\n schemaLocation\n 4\n true\n scope,groupId,artifactId\n false\n false\n \n \n \n \n verify\n \n validate\n \n STOP\n \n \n \n \n \n com.diffplug.spotless\n spotless-maven-plugin\n ${spotless-maven-plugin.version}\n \n \n \n \n **/*.kt\n **/*.kts\n \n \n 0.46\n \n 2\n 4\n true\n 100\n \n \n \n \n \n \n check\n \n compile\n \n \n \n \n ${basedir}\n \n\n" }, { "path": ".teamcity/settings.kts", "content": "import builds.Build\nimport builds.DEFAULT_BRANCH\nimport builds.JavaVersion\nimport builds.Neo4jSparkConnectorVcs\nimport builds.Neo4jVersion\nimport builds.PySparkVersion\nimport builds.ScalaVersion\nimport jetbrains.buildServer.configs.kotlin.Project\nimport jetbrains.buildServer.configs.kotlin.failureConditions.BuildFailureOnText\nimport jetbrains.buildServer.configs.kotlin.failureConditions.failOnText\nimport jetbrains.buildServer.configs.kotlin.project\nimport jetbrains.buildServer.configs.kotlin.triggers.schedule\nimport jetbrains.buildServer.configs.kotlin.triggers.vcs\nimport jetbrains.buildServer.configs.kotlin.version\n\nversion = \"2025.11\"\n\nproject {\n params {\n text(\"default-spark-branch\", DEFAULT_BRANCH)\n\n text(\"osssonatypeorg-username\", \"%publish-username%\")\n password(\"osssonatypeorg-password\", \"%publish-password%\")\n password(\"signing-key-passphrase\", \"%publish-signing-key-password%\")\n password(\"github-commit-status-token\", \"%github-token%\")\n password(\"github-pull-request-token\", \"%github-token%\")\n password(\"semgrep-app-token\", \"%semgrep-token%\")\n }\n\n vcsRoot(Neo4jSparkConnectorVcs)\n\n subProject(\n Build(\n name = \"main\",\n javaVersions =\n setOf(JavaVersion.V_8, JavaVersion.V_11, JavaVersion.V_17, JavaVersion.V_21),\n scalaVersions = setOf(ScalaVersion.V2_12, ScalaVersion.V2_13),\n pysparkVersions = setOf(PySparkVersion.V3_4, PySparkVersion.V3_5),\n neo4jVersions = setOf(Neo4jVersion.V_4_4, Neo4jVersion.V_5, Neo4jVersion.V_CALVER),\n forPullRequests = false,\n ) {\n triggers {\n vcs {\n this.branchFilter = buildString {\n appendLine(\"+:$DEFAULT_BRANCH\")\n appendLine(\"+:refs/heads/$DEFAULT_BRANCH\")\n }\n this.triggerRules =\n \"\"\"\n -:comment=^build.*release version.*:**\n -:comment=^build.*update version.*:**\n \"\"\"\n .trimIndent()\n }\n }\n },\n )\n\n subProject(\n Build(\n name = \"pull-request\",\n javaVersions = setOf(JavaVersion.V_8, JavaVersion.V_11, JavaVersion.V_17),\n scalaVersions = setOf(ScalaVersion.V2_12, ScalaVersion.V2_13),\n pysparkVersions = setOf(PySparkVersion.V3_5),\n neo4jVersions = setOf(Neo4jVersion.V_4_4, Neo4jVersion.V_5, Neo4jVersion.V_CALVER),\n forPullRequests = true,\n ) {\n triggers {\n vcs {\n this.branchFilter = buildString {\n appendLine(\"+:pull/*\")\n appendLine(\"+:refs/heads/pull/*\")\n }\n }\n }\n\n // when a PR gets closed, TC falls back to main branch to run the pipeline, which we don't\n // want\n failureConditions {\n failOnText {\n conditionType = BuildFailureOnText.ConditionType.CONTAINS\n pattern = \"which does not correspond to any branch monitored by the build VCS roots\"\n failureMessage = \"Error: The branch %teamcity.build.branch% does not exist\"\n reverse = false\n stopBuildOnFailure = true\n }\n }\n },\n )\n\n subProject(\n Project {\n this.id(\"compatibility\")\n name = \"compatibility\"\n\n Neo4jVersion.entries.minus(Neo4jVersion.V_NONE).forEach { neo4j ->\n subProject(\n Build(\n name = neo4j.version,\n javaVersions =\n setOf(JavaVersion.V_8, JavaVersion.V_11, JavaVersion.V_17, JavaVersion.V_21),\n scalaVersions = setOf(ScalaVersion.V2_12, ScalaVersion.V2_13),\n pysparkVersions = setOf(PySparkVersion.V3_4, PySparkVersion.V3_5),\n neo4jVersions = setOf(neo4j),\n forPullRequests = false,\n forCompatibility = true,\n ) {\n triggers {\n vcs { enabled = false }\n\n schedule {\n branchFilter = buildString {\n appendLine(\"+:$DEFAULT_BRANCH\")\n appendLine(\"+:refs/heads/$DEFAULT_BRANCH\")\n }\n schedulingPolicy = daily {\n hour = 7\n minute = 0\n }\n triggerBuild = always()\n withPendingChangesOnly = false\n enforceCleanCheckout = true\n enforceCleanCheckoutForDependencies = true\n }\n }\n },\n )\n }\n },\n )\n}\n" }, { "path": "LICENSE.txt", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "# Neo4j Connector for Apache Spark\n\nThis repository contains the Neo4j Connector for Apache Spark.\n\n## License\n\nThis neo4j-connector-apache-spark is Apache 2 Licensed\n\n## Documentation\n\nThe documentation for Neo4j Connector for Apache Spark lives at https://github.com/neo4j/docs-spark repository.\n\n## Building for Spark 3\n\nYou can build for Spark 3.x with both Scala 2.12 and Scala 2.13\n\n```\n./maven-release.sh package 2.12\n./maven-release.sh package 2.13\n```\n\nThese commands will generate the corresponding targets\n* `spark-3/target/neo4j-connector-apache-spark_2.12-_for_spark_3.jar`\n* `spark-3/target/neo4j-connector-apache-spark_2.13-_for_spark_3.jar`\n\n\n## Integration with Apache Spark Applications\n\n**spark-shell, pyspark, or spark-submit**\n\n`$SPARK_HOME/bin/spark-shell --jars neo4j-connector-apache-spark_2.12-_for_spark_3.jar`\n\n`$SPARK_HOME/bin/spark-shell --packages org.neo4j:neo4j-connector-apache-spark_2.12:_for_spark_3`\n\n**sbt**\n\nIf you use the [sbt-spark-package plugin](https://github.com/databricks/sbt-spark-package), in your sbt build file, add:\n\n```scala\nresolvers += \"Spark Packages Repo\" at \"http://dl.bintray.com/spark-packages/maven\"\nlibraryDependencies += \"org.neo4j\" % \"neo4j-connector-apache-spark_2.12\" % \"_for_spark_3\"\n``` \n\n**maven** \n\nIn your pom.xml, add: \n\n```xml\n\n \n \n org.neo4j\n neo4j-connector-apache-spark_2.12\n [version]_for_spark_3\n \n\n```\n\nFor more info about the available version visit https://neo4j.com/developer/spark/overview/#_compatibility\n" }, { "path": "common/LICENSES.txt", "content": "This file contains the full license text of the included third party\nlibraries. For an overview of the licenses see the NOTICE.txt file.\n\n\n------------------------------------------------------------------------------\nApache Software License, Version 2.0\n IntelliJ IDEA Annotations\n Kotlin Stdlib\n Netty/Buffer\n Netty/Codec\n Netty/Common\n Netty/Handler\n Netty/Resolver\n Netty/TomcatNative [OpenSSL - Classes]\n Netty/Transport\n Netty/Transport/Native/Unix/Common\n Non-Blocking Reactive Foundation for the JVM\n org.apiguardian:apiguardian-api\n------------------------------------------------------------------------------\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n\n\n------------------------------------------------------------------------------\nMIT License\n SLF4J API Module\n------------------------------------------------------------------------------\n\nThe MIT License\n\nCopyright (c) \n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n\n\n\n------------------------------------------------------------------------------\nMIT No Attribution License\n reactive-streams\n------------------------------------------------------------------------------\n\nMIT No Attribution\n\nCopyright \n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this\nsoftware and associated documentation files (the \"Software\"), to deal in the Software\nwithout restriction, including without limitation the rights to use, copy, modify,\nmerge, publish, distribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,\nINCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT\nHOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\n\n\n\n" }, { "path": "common/NOTICE.txt", "content": "Copyright (c) \"Neo4j\"\nNeo4j Sweden AB [https://neo4j.com]\n\nThis file is part of Neo4j.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nFull license texts are found in LICENSES.txt.\n\n\nThird-party licenses\n--------------------\n\nApache Software License, Version 2.0\n IntelliJ IDEA Annotations\n Kotlin Stdlib\n Netty/Buffer\n Netty/Codec\n Netty/Common\n Netty/Handler\n Netty/Resolver\n Netty/TomcatNative [OpenSSL - Classes]\n Netty/Transport\n Netty/Transport/Native/Unix/Common\n Non-Blocking Reactive Foundation for the JVM\n org.apiguardian:apiguardian-api\n\nMIT License\n SLF4J API Module\n\nMIT No Attribution License\n reactive-streams\n\n" }, { "path": "common/pom.xml", "content": "\n\n 4.0.0\n \n org.neo4j\n neo4j-connector-apache-spark_parent\n 5.4.3-SNAPSHOT\n \n neo4j-connector-apache-spark_common\n jar\n neo4j-connector-apache-spark-common\n Common Services for Neo4j Connector for Apache Spark using the binary Bolt Driver\n \n \n org.neo4j\n caniuse-core\n \n \n org.neo4j\n caniuse-neo4j-detection\n \n \n org.neo4j\n neo4j-cypher-dsl\n \n \n org.neo4j.connectors\n commons-authn-spi\n \n \n org.neo4j.connectors\n commons-reauth-driver\n \n \n org.neo4j.driver\n neo4j-java-driver-slim\n \n \n org.apache.spark\n spark-core_${scala.binary.version}\n provided\n \n \n org.apache.spark\n spark-sql_${scala.binary.version}\n provided\n \n \n org.scala-lang\n scala-library\n provided\n \n \n org.scala-lang\n scala-reflect\n provided\n \n \n org.neo4j.connectors\n commons-authn-provided\n runtime\n \n \n org.neo4j\n neo4j-connector-apache-spark_test-support\n ${project.version}\n test\n \n \n org.scalatest\n scalatest_${scala.binary.version}\n test\n \n \n org.scalatestplus\n junit-4-13_${scala.binary.version}\n test\n \n \n pl.pragmatists\n JUnitParams\n 1.1.1\n test\n \n \n \n \n \n true\n src/main/resources\n \n \n \n \n net.alchim31.maven\n scala-maven-plugin\n \n \n org.apache.maven.plugins\n maven-failsafe-plugin\n \n \n org.apache.maven.plugins\n maven-surefire-plugin\n \n \n \n\n" }, { "path": "common/src/main/java/org/neo4j/spark/util/ReflectionUtils.java", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util;\n\nimport org.apache.spark.sql.connector.expressions.Expression;\nimport org.apache.spark.sql.connector.expressions.NamedReference;\nimport org.apache.spark.sql.connector.expressions.aggregate.Aggregation;\n\nimport java.lang.invoke.MethodHandle;\nimport java.lang.invoke.MethodHandles;\nimport java.lang.invoke.MethodType;\nimport java.util.Optional;\nimport java.util.stream.Stream;\n\npublic class ReflectionUtils {\n\n private static final MethodHandles.Lookup lookup = MethodHandles.lookup();\n\n private static Optional getGroupByColumns() {\n try {\n return Optional.of(lookup\n .findVirtual(Aggregation.class, \"groupByColumns\", MethodType.methodType(NamedReference[].class))\n .asType(MethodType.methodType(Expression[].class, Aggregation.class)));\n } catch (Exception e) {\n return Optional.empty();\n }\n }\n\n private static Optional getGroupByExpressions() {\n try {\n return Optional.of(lookup\n .findVirtual(Aggregation.class, \"groupByExpressions\", MethodType.methodType(Expression[].class)));\n } catch (Exception e) {\n return Optional.empty();\n }\n }\n\n private static final Optional groupByColumns = getGroupByColumns();\n private static final Optional groupByExpressions = getGroupByExpressions();\n\n private static final Expression[] EMPTY = new Expression[0];\n\n public static Expression[] groupByCols(Aggregation agg) {\n return Stream.of(groupByExpressions, groupByColumns)\n .filter(Optional::isPresent)\n .map(Optional::get)\n .map(mh -> {\n try {\n return (Expression[]) mh.invokeExact(agg);\n } catch (Throwable e) {\n return EMPTY;\n }\n })\n .findFirst()\n .orElse(EMPTY);\n }\n\n}" }, { "path": "common/src/main/resources/neo4j-spark-connector.properties", "content": "version=${project.version}" }, { "path": "common/src/main/scala/org/neo4j/spark/config/TopN.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.config\n\nimport org.apache.spark.sql.connector.expressions.SortOrder\n\ncase class TopN(limit: Long, orders: Array[SortOrder] = Array.empty)\n" }, { "path": "common/src/main/scala/org/neo4j/spark/converter/DataConverter.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.converter\n\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.catalyst.expressions.GenericRow\nimport org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema\nimport org.apache.spark.sql.catalyst.expressions.UnsafeRow\nimport org.apache.spark.sql.catalyst.util.ArrayBasedMapData\nimport org.apache.spark.sql.catalyst.util.ArrayData\nimport org.apache.spark.sql.catalyst.util.DateTimeUtils\nimport org.apache.spark.sql.catalyst.util.MapData\nimport org.apache.spark.sql.types._\nimport org.apache.spark.unsafe.types.UTF8String\nimport org.neo4j.driver.Value\nimport org.neo4j.driver.Values\nimport org.neo4j.driver.internal._\nimport org.neo4j.driver.types.IsoDuration\nimport org.neo4j.driver.types.Node\nimport org.neo4j.driver.types.Relationship\nimport org.neo4j.spark.service.SchemaService\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.Neo4jUtil\n\nimport java.time._\nimport java.time.format.DateTimeFormatter\n\nimport scala.annotation.tailrec\nimport scala.collection.JavaConverters._\n\ntrait DataConverter[T] {\n def convert(value: Any, dataType: DataType = null): T\n\n @tailrec\n final private[converter] def extractStructType(dataType: DataType): StructType = dataType match {\n case structType: StructType => structType\n case mapType: MapType => extractStructType(mapType.valueType)\n case arrayType: ArrayType => extractStructType(arrayType.elementType)\n case _ => throw new UnsupportedOperationException(s\"$dataType not supported\")\n }\n}\n\nobject SparkToNeo4jDataConverter {\n def apply(options: Neo4jOptions): SparkToNeo4jDataConverter = new SparkToNeo4jDataConverter(options)\n\n private def dayTimeMicrosToNeo4jDuration(micros: Long): Value = {\n val oneSecondInMicros = 1000000L\n val oneDayInMicros = 24 * 3600 * oneSecondInMicros\n val numberDays = Math.floorDiv(micros, oneDayInMicros)\n val remainderMicros = Math.floorMod(micros, oneDayInMicros)\n val numberSeconds = Math.floorDiv(remainderMicros, oneSecondInMicros)\n val numberNanos = Math.floorMod(remainderMicros, oneSecondInMicros) * 1000\n Values.isoDuration(0L, numberDays, numberSeconds, numberNanos.toInt)\n }\n\n // while Neo4j supports years, this driver version's API does not expose it.\n private def yearMonthIntervalToNeo4jDuration(months: Int): Value = {\n Values.isoDuration(months.toLong, 0L, 0L, 0)\n }\n}\n\nclass SparkToNeo4jDataConverter(options: Neo4jOptions) extends DataConverter[Value] {\n\n override def convert(value: Any, dataType: DataType): Value = {\n value match {\n case date: java.sql.Date => convert(date.toLocalDate, dataType)\n case timestamp: java.sql.Timestamp =>\n if (options.legacyTypeConversionEnabled) {\n convert(timestamp.toLocalDateTime, dataType)\n } else {\n convert(timestamp.toInstant.atZone(ZoneOffset.UTC), dataType)\n }\n case intValue: Int if dataType == DataTypes.DateType =>\n convert(\n DateTimeUtils\n .toJavaDate(intValue),\n dataType\n )\n case intValue: Int if dataType.isInstanceOf[YearMonthIntervalType] && !options.legacyTypeConversionEnabled =>\n SparkToNeo4jDataConverter.yearMonthIntervalToNeo4jDuration(intValue)\n case longValue: Long if dataType == DataTypes.TimestampType =>\n convert(DateTimeUtils.toJavaTimestamp(longValue), dataType)\n case longValue: Long if dataType == DataTypes.TimestampNTZType && !options.legacyTypeConversionEnabled =>\n convert(DateTimeUtils.microsToLocalDateTime(longValue), dataType)\n case longValue: Long if dataType.isInstanceOf[DayTimeIntervalType] && !options.legacyTypeConversionEnabled =>\n SparkToNeo4jDataConverter.dayTimeMicrosToNeo4jDuration(longValue)\n case unsafeRow: UnsafeRow => {\n val structType = extractStructType(dataType)\n val row = new GenericRowWithSchema(unsafeRow.toSeq(structType).toArray, structType)\n convert(row)\n }\n case struct: GenericRow => {\n def toMap(struct: GenericRow): Value = {\n Values.value(\n struct.schema.fields.map(f => f.name -> convert(struct.getAs(f.name), f.dataType)).toMap.asJava\n )\n }\n\n try {\n struct.getAs[UTF8String](\"type\").toString match {\n case SchemaService.POINT_TYPE_2D => Values.point(\n struct.getAs[Number](\"srid\").intValue(),\n struct.getAs[Number](\"x\").doubleValue(),\n struct.getAs[Number](\"y\").doubleValue()\n )\n case SchemaService.POINT_TYPE_3D => Values.point(\n struct.getAs[Number](\"srid\").intValue(),\n struct.getAs[Number](\"x\").doubleValue(),\n struct.getAs[Number](\"y\").doubleValue(),\n struct.getAs[Number](\"z\").doubleValue()\n )\n case SchemaService.DURATION_TYPE => Values.isoDuration(\n struct.getAs[Number](\"months\").longValue(),\n struct.getAs[Number](\"days\").longValue(),\n struct.getAs[Number](\"seconds\").longValue(),\n struct.getAs[Number](\"nanoseconds\").intValue()\n )\n case SchemaService.TIME_TYPE_OFFSET =>\n Values.value(OffsetTime.parse(struct.getAs[UTF8String](\"value\").toString))\n case SchemaService.TIME_TYPE_LOCAL =>\n Values.value(LocalTime.parse(struct.getAs[UTF8String](\"value\").toString))\n case _ => toMap(struct)\n }\n } catch {\n case _: Throwable => toMap(struct)\n }\n }\n case unsafeArray: ArrayData => {\n val sparkType = dataType match {\n case arrayType: ArrayType => arrayType.elementType\n case _ => dataType\n }\n if (sparkType == DataTypes.ByteType && !options.legacyTypeConversionEnabled) {\n Values.value(unsafeArray.toByteArray)\n } else {\n val javaList = unsafeArray.toSeq[AnyRef](sparkType)\n .map(elem => convert(elem, sparkType))\n .asJava\n Values.value(javaList)\n }\n }\n case unsafeMapData: MapData => { // Neo4j only supports Map[String, AnyRef]\n val mapType = dataType.asInstanceOf[MapType]\n val map: Map[String, AnyRef] = (0 until unsafeMapData.numElements())\n .map(i =>\n (unsafeMapData.keyArray().getUTF8String(i).toString, unsafeMapData.valueArray().get(i, mapType.valueType))\n )\n .toMap[String, AnyRef]\n .mapValues(innerValue => convert(innerValue, mapType.valueType))\n .toMap[String, AnyRef]\n Values.value(map.asJava)\n }\n case string: UTF8String => convert(string.toString)\n case decimal: Decimal if dataType.isInstanceOf[DecimalType] => Values.value(decimal.toString)\n case _ => Values.value(value)\n }\n }\n}\n\nobject Neo4jToSparkDataConverter {\n def apply(options: Neo4jOptions): Neo4jToSparkDataConverter = new Neo4jToSparkDataConverter(options)\n}\n\nclass Neo4jToSparkDataConverter(options: Neo4jOptions) extends DataConverter[Any] {\n\n override def convert(value: Any, dataType: DataType): Any = {\n if (dataType != null && dataType == DataTypes.StringType && value != null && !value.isInstanceOf[String]) {\n convert(Neo4jUtil.mapper.writeValueAsString(value), dataType)\n } else {\n value match {\n case node: Node => {\n val map = node.asMap()\n val structType = extractStructType(dataType)\n val fields = structType\n .filter(field => field.name != Neo4jUtil.INTERNAL_ID_FIELD && field.name != Neo4jUtil.INTERNAL_LABELS_FIELD)\n .map(field => convert(map.get(field.name), field.dataType))\n InternalRow.fromSeq(Seq(convert(node.id()), convert(node.labels())) ++ fields)\n }\n case rel: Relationship => {\n val map = rel.asMap()\n val structType = extractStructType(dataType)\n val fields = structType\n .filter(field =>\n field.name != Neo4jUtil.INTERNAL_REL_ID_FIELD\n && field.name != Neo4jUtil.INTERNAL_REL_TYPE_FIELD\n && field.name != Neo4jUtil.INTERNAL_REL_SOURCE_ID_FIELD\n && field.name != Neo4jUtil.INTERNAL_REL_TARGET_ID_FIELD\n )\n .map(field => convert(map.get(field.name), field.dataType))\n InternalRow.fromSeq(Seq(\n convert(rel.id()),\n convert(rel.`type`()),\n convert(rel.startNodeId()),\n convert(rel.endNodeId())\n ) ++ fields)\n }\n case d: IsoDuration => {\n val months = d.months()\n val days = d.days()\n val nanoseconds: Integer = d.nanoseconds()\n val seconds = d.seconds()\n InternalRow.fromSeq(Seq(\n UTF8String.fromString(SchemaService.DURATION_TYPE),\n months,\n days,\n seconds,\n nanoseconds,\n UTF8String.fromString(d.toString)\n ))\n }\n case zt: ZonedDateTime => DateTimeUtils.instantToMicros(zt.toInstant)\n case dt: LocalDateTime => {\n if (options.legacyTypeConversionEnabled) {\n DateTimeUtils.instantToMicros(dt.toInstant(ZoneOffset.UTC))\n } else {\n DateTimeUtils.localDateTimeToMicros(dt)\n }\n }\n case d: LocalDate => d.toEpochDay.toInt\n case lt: LocalTime => {\n InternalRow.fromSeq(Seq(\n UTF8String.fromString(SchemaService.TIME_TYPE_LOCAL),\n UTF8String.fromString(lt.format(DateTimeFormatter.ISO_TIME))\n ))\n }\n case t: OffsetTime => {\n InternalRow.fromSeq(Seq(\n UTF8String.fromString(SchemaService.TIME_TYPE_OFFSET),\n UTF8String.fromString(t.format(DateTimeFormatter.ISO_TIME))\n ))\n }\n case p: InternalPoint2D => {\n val srid: Integer = p.srid()\n InternalRow.fromSeq(Seq(UTF8String.fromString(SchemaService.POINT_TYPE_2D), srid, p.x(), p.y(), null))\n }\n case p: InternalPoint3D => {\n val srid: Integer = p.srid()\n InternalRow.fromSeq(Seq(UTF8String.fromString(SchemaService.POINT_TYPE_3D), srid, p.x(), p.y(), p.z()))\n }\n case l: java.util.List[_] => {\n val elementType = if (dataType != null) dataType.asInstanceOf[ArrayType].elementType else null\n ArrayData.toArrayData(l.asScala.map(e => convert(e, elementType)).toArray)\n }\n case map: java.util.Map[_, _] => {\n if (dataType != null) {\n val mapType = dataType.asInstanceOf[MapType]\n ArrayBasedMapData(map.asScala.map(t => (convert(t._1, mapType.keyType), convert(t._2, mapType.valueType))))\n } else {\n ArrayBasedMapData(map.asScala.map(t => (convert(t._1), convert(t._2))))\n }\n }\n case s: String => UTF8String.fromString(s)\n case _ => value\n }\n }\n }\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/converter/TypeConverter.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.converter\n\nimport org.apache.spark.sql.types.DataType\nimport org.apache.spark.sql.types.DataTypes\nimport org.apache.spark.sql.types.DayTimeIntervalType\nimport org.apache.spark.sql.types.DecimalType\nimport org.apache.spark.sql.types.YearMonthIntervalType\nimport org.neo4j.driver.types.Entity\nimport org.neo4j.spark.converter.CypherToSparkTypeConverter.cleanTerms\nimport org.neo4j.spark.converter.CypherToSparkTypeConverter.durationType\nimport org.neo4j.spark.converter.CypherToSparkTypeConverter.pointType\nimport org.neo4j.spark.converter.CypherToSparkTypeConverter.timeType\nimport org.neo4j.spark.converter.SparkToCypherTypeConverter.mapping\nimport org.neo4j.spark.service.SchemaService.normalizedClassName\nimport org.neo4j.spark.util.Neo4jImplicits.EntityImplicits\nimport org.neo4j.spark.util.Neo4jOptions\n\nimport scala.collection.JavaConverters._\n\ntrait TypeConverter[SOURCE_TYPE, DESTINATION_TYPE] {\n def convert(sourceType: SOURCE_TYPE, value: Any = null): DESTINATION_TYPE\n}\n\nobject CypherToSparkTypeConverter {\n def apply(options: Neo4jOptions): CypherToSparkTypeConverter = new CypherToSparkTypeConverter(options)\n\n private val cleanTerms: String = \"Unmodifiable|Internal|Iso|2D|3D|Offset\"\n\n val durationType: DataType = DataTypes.createStructType(Array(\n DataTypes.createStructField(\"type\", DataTypes.StringType, false),\n DataTypes.createStructField(\"months\", DataTypes.LongType, false),\n DataTypes.createStructField(\"days\", DataTypes.LongType, false),\n DataTypes.createStructField(\"seconds\", DataTypes.LongType, false),\n DataTypes.createStructField(\"nanoseconds\", DataTypes.IntegerType, false),\n DataTypes.createStructField(\"value\", DataTypes.StringType, false)\n ))\n\n val pointType: DataType = DataTypes.createStructType(Array(\n DataTypes.createStructField(\"type\", DataTypes.StringType, false),\n DataTypes.createStructField(\"srid\", DataTypes.IntegerType, false),\n DataTypes.createStructField(\"x\", DataTypes.DoubleType, false),\n DataTypes.createStructField(\"y\", DataTypes.DoubleType, false),\n DataTypes.createStructField(\"z\", DataTypes.DoubleType, true)\n ))\n\n val timeType: DataType = DataTypes.createStructType(Array(\n DataTypes.createStructField(\"type\", DataTypes.StringType, false),\n DataTypes.createStructField(\"value\", DataTypes.StringType, false)\n ))\n}\n\nclass CypherToSparkTypeConverter(options: Neo4jOptions) extends TypeConverter[String, DataType] {\n\n override def convert(sourceType: String, value: Any = null): DataType = {\n var cleanedSourceType = sourceType.replaceAll(cleanTerms, \"\")\n if (options.legacyTypeConversionEnabled) {\n cleanedSourceType = cleanedSourceType.replaceAll(\"Local|Zoned\", \"\")\n }\n cleanedSourceType match {\n case \"Node\" | \"Relationship\" =>\n if (value != null) value.asInstanceOf[Entity].toStruct(options) else DataTypes.NullType\n case \"NodeArray\" | \"RelationshipArray\" =>\n if (value != null) DataTypes.createArrayType(value.asInstanceOf[Entity].toStruct(options))\n else DataTypes.NullType\n case \"Boolean\" => DataTypes.BooleanType\n case \"Long\" => DataTypes.LongType\n case \"Double\" => DataTypes.DoubleType\n case \"Point\" => pointType\n case \"DateTime\" | \"ZonedDateTime\" => DataTypes.TimestampType\n case \"LocalDateTime\" =>\n if (options.legacyTypeConversionEnabled) {\n DataTypes.TimestampType\n } else {\n DataTypes.TimestampNTZType\n }\n case \"Time\" | \"LocalTime\" => timeType\n case \"Date\" | \"LocalDate\" => DataTypes.DateType\n case \"Duration\" => durationType\n case \"ByteArray\" => DataTypes.BinaryType\n case \"Map\" => {\n val valueType = if (value == null) {\n DataTypes.NullType\n } else {\n val map = value.asInstanceOf[java.util.Map[String, AnyRef]].asScala\n val types = map.values\n .map(value => normalizedClassName(value, options))\n .toSet\n if (types.size == 1) convert(types.head, map.values.head) else DataTypes.StringType\n }\n DataTypes.createMapType(DataTypes.StringType, valueType)\n }\n case \"Array\" => {\n val valueType = if (value == null) {\n DataTypes.NullType\n } else {\n val list = value.asInstanceOf[java.util.List[AnyRef]].asScala\n val types = list\n .map(value => normalizedClassName(value, options))\n .toSet\n if (types.size == 1) convert(types.head, list.head) else DataTypes.StringType\n }\n DataTypes.createArrayType(valueType)\n }\n // These are from APOC\n case \"StringArray\" => DataTypes.createArrayType(DataTypes.StringType)\n case \"LongArray\" => DataTypes.createArrayType(DataTypes.LongType)\n case \"DoubleArray\" => DataTypes.createArrayType(DataTypes.DoubleType)\n case \"BooleanArray\" => DataTypes.createArrayType(DataTypes.BooleanType)\n case \"PointArray\" => DataTypes.createArrayType(pointType)\n case \"DateTimeArray\" | \"ZonedDateTimeArray\" => DataTypes.createArrayType(DataTypes.TimestampType)\n case \"TimeArray\" | \"LocalTimeArray\" => DataTypes.createArrayType(timeType)\n case \"DateArray\" | \"LocalDateArray\" => DataTypes.createArrayType(DataTypes.DateType)\n case \"DurationArray\" => DataTypes.createArrayType(durationType)\n // Default is String\n case _ => DataTypes.StringType\n }\n }\n}\n\nobject SparkToCypherTypeConverter {\n def apply(options: Neo4jOptions): SparkToCypherTypeConverter = new SparkToCypherTypeConverter(options)\n\n private val baseMappings: Map[DataType, String] = Map(\n DataTypes.BooleanType -> \"BOOLEAN\",\n DataTypes.StringType -> \"STRING\",\n DecimalType.SYSTEM_DEFAULT -> \"STRING\",\n DataTypes.ByteType -> \"INTEGER\",\n DataTypes.ShortType -> \"INTEGER\",\n DataTypes.IntegerType -> \"INTEGER\",\n DataTypes.LongType -> \"INTEGER\",\n DataTypes.FloatType -> \"FLOAT\",\n DataTypes.DoubleType -> \"FLOAT\",\n DataTypes.DateType -> \"DATE\",\n durationType -> \"DURATION\",\n pointType -> \"POINT\",\n // Cypher graph entities do not allow null values in arrays\n DataTypes.createArrayType(DataTypes.BooleanType, false) -> \"LIST\",\n DataTypes.createArrayType(DataTypes.StringType, false) -> \"LIST\",\n DataTypes.createArrayType(DecimalType.SYSTEM_DEFAULT, false) -> \"LIST\",\n DataTypes.createArrayType(DataTypes.ShortType, false) -> \"LIST\",\n DataTypes.createArrayType(DataTypes.IntegerType, false) -> \"LIST\",\n DataTypes.createArrayType(DataTypes.LongType, false) -> \"LIST\",\n DataTypes.createArrayType(DataTypes.FloatType, false) -> \"LIST\",\n DataTypes.createArrayType(DataTypes.DoubleType, false) -> \"LIST\",\n DataTypes.createArrayType(DataTypes.DateType, false) -> \"LIST\",\n DataTypes.createArrayType(durationType, false) -> \"LIST\",\n DataTypes.createArrayType(pointType, false) -> \"LIST\"\n )\n\n private def mapping(sourceType: DataType, options: Neo4jOptions): String = {\n val mappings = sourceTypeMappings(options)\n mappings(sourceType)\n }\n\n private def sourceTypeMappings(options: Neo4jOptions): Map[DataType, String] = {\n var result = baseMappings\n if (options.legacyTypeConversionEnabled) {\n result += (DataTypes.TimestampType -> \"LOCAL DATETIME\")\n result += (DataTypes.createArrayType(DataTypes.TimestampType, false) -> \"LIST\")\n result += (DataTypes.createArrayType(DataTypes.TimestampType, true) -> \"LIST\")\n } else {\n result += (DataTypes.TimestampType -> \"ZONED DATETIME\")\n result += (DataTypes.TimestampNTZType -> \"LOCAL DATETIME\")\n result += (DayTimeIntervalType() -> \"DURATION\")\n result += (YearMonthIntervalType() -> \"DURATION\")\n result += (DataTypes.createArrayType(DataTypes.ByteType, false) -> \"ByteArray\")\n result += (DataTypes.createArrayType(DataTypes.TimestampType, false) -> \"LIST\")\n result += (DataTypes.createArrayType(DataTypes.TimestampNTZType, false) -> \"LIST\")\n result += (DataTypes.createArrayType(DayTimeIntervalType(), false) -> \"LIST\")\n result += (DataTypes.createArrayType(DayTimeIntervalType(), true) -> \"LIST\")\n result += (DataTypes.createArrayType(YearMonthIntervalType(), false) -> \"LIST\")\n result += (DataTypes.createArrayType(YearMonthIntervalType(), true) -> \"LIST\")\n }\n result\n }\n}\n\nclass SparkToCypherTypeConverter(options: Neo4jOptions) extends TypeConverter[DataType, String] {\n override def convert(sourceType: DataType, value: Any): String = mapping(sourceType, options)\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/cypher/Cypher5Renderer.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.cypher\n\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.caniuse.Neo4jVersion\nimport org.neo4j.cypherdsl.core.Statement\nimport org.neo4j.cypherdsl.core.renderer.Configuration\nimport org.neo4j.cypherdsl.core.renderer.Dialect\nimport org.neo4j.cypherdsl.core.renderer.Renderer\nimport org.neo4j.spark.cypher.Cypher5Renderer.Neo4jV5\nimport org.neo4j.spark.cypher.CypherVersionSelector.selectCypherVersionClause\n\nclass Cypher5Renderer(neo4j: Neo4j) extends Renderer {\n\n private val delegate =\n Renderer.getRenderer(\n Configuration.newConfig()\n .withDialect(\n if (neo4j.getVersion.compareTo(Neo4jV5) < 0) {\n Dialect.DEFAULT\n } else {\n Dialect.NEO4J_5\n }\n )\n .build()\n )\n\n override def render(statement: Statement): String = {\n val rendered = delegate.render(statement)\n s\"${selectCypherVersionClause(neo4j)}$rendered\"\n }\n\n}\n\nprivate object Cypher5Renderer {\n private val Neo4jV5 = new Neo4jVersion(5, 0, 0)\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/cypher/CypherVersionSelector.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.cypher\n\nimport org.neo4j.caniuse.CanIUse.INSTANCE.canIUse\nimport org.neo4j.caniuse.Cypher.{INSTANCE => Cypher}\nimport org.neo4j.caniuse.Neo4j\n\nobject CypherVersionSelector {\n\n def selectCypherVersionClause(neo4j: Neo4j): String = {\n if (canIUse(Cypher.explicitCypher5Selection()).withNeo4j(neo4j)) {\n \"CYPHER 5 \"\n } else {\n \"\"\n }\n }\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/reader/BasePartitionReader.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.reader\n\nimport org.apache.spark.internal.Logging\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.connector.expressions.aggregate.AggregateFunc\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.driver.Record\nimport org.neo4j.driver.Session\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.Values\nimport org.neo4j.spark.service.MappingService\nimport org.neo4j.spark.service.Neo4jQueryReadStrategy\nimport org.neo4j.spark.service.Neo4jQueryService\nimport org.neo4j.spark.service.Neo4jQueryStrategy\nimport org.neo4j.spark.service.Neo4jReadMappingStrategy\nimport org.neo4j.spark.service.PartitionPagination\nimport org.neo4j.spark.util.DriverCache\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.Neo4jUtil\nimport org.neo4j.spark.util.QueryType\n\nimport java.io.IOException\nimport java.time.Duration\nimport java.util\nimport java.util.concurrent.atomic.AtomicInteger\nimport java.util.concurrent.locks.LockSupport\n\nimport scala.collection.JavaConverters._\n\nabstract class BasePartitionReader(\n private val neo4j: Neo4j,\n private val options: Neo4jOptions,\n private val filters: Array[Filter],\n private val schema: StructType,\n private val jobId: String,\n private val partitionSkipLimit: PartitionPagination,\n private val scriptResult: java.util.List[java.util.Map[String, AnyRef]],\n private val requiredColumns: StructType,\n private val aggregateColumns: Array[AggregateFunc]\n) extends Logging {\n private var result: Iterator[Record] = _\n private var session: Session = _\n private var transaction: Transaction = _\n\n protected val name: String =\n if (partitionSkipLimit.partitionNumber > 0) s\"$jobId-${partitionSkipLimit.partitionNumber}\" else jobId\n protected val driverCache: DriverCache = new DriverCache(options.connection)\n\n private var nextRow: InternalRow = _\n\n private lazy val values = {\n val params = new java.util.HashMap[String, Any]()\n params.put(Neo4jQueryStrategy.VARIABLE_SCRIPT_RESULT, scriptResult)\n Neo4jUtil.paramsFromFilters(filters)\n .foreach(p => params.put(p._1, p._2))\n\n if (options.query.queryType == QueryType.GDS) {\n params.putAll(options.gdsMetadata.parameters)\n }\n\n params\n }\n\n private val mappingService = new MappingService(new Neo4jReadMappingStrategy(options, requiredColumns), options)\n\n @volatile\n private var error: Boolean = false\n\n private val retries = new AtomicInteger(options.transactionSettings.retries)\n\n @throws(classOf[IOException])\n def next: Boolean =\n try {\n nextHandler()\n } catch {\n case t: Throwable =>\n if (options.transactionSettings.shouldFailOn(t)) {\n error = true\n logError(\"Error while invoking next due to explicitly configured failure condition:\", t)\n throw new IOException(t)\n }\n\n if (Neo4jUtil.isRetryableException(t) && retries.get() > 0) {\n val currentRetry = retries.decrementAndGet\n logInfo(\n s\"encountered a transient exception while reading, retrying ${options.transactionSettings.retries - currentRetry} time(s)\",\n t\n )\n\n close()\n result = null // Reset result to force new query\n\n // Wait before retry\n LockSupport.parkNanos(Duration.ofMillis(options.transactionSettings.retryTimeout).toNanos)\n next\n } else {\n error = true\n logError(\"Error while invoking next:\", t)\n throw new IOException(t)\n }\n }\n\n private def nextHandler(): Boolean = {\n if (result == null) {\n session = driverCache.getOrCreate().session(options.session.toNeo4jSession())\n transaction = session.beginTransaction(options.toNeo4jTransactionConfig)\n\n val queryText = query()\n val queryParams = queryParameters\n\n logInfo(s\"Running the following query on Neo4j: $queryText\")\n logDebug(s\"with parameters $queryParams\")\n\n result = transaction.run(queryText, Values.value(queryParams))\n .asScala\n }\n\n if (result.hasNext) {\n nextRow = mappingService.convert(result.next(), schema)\n true\n } else {\n false\n }\n }\n\n def get: InternalRow = nextRow\n\n def close(): Unit = {\n Neo4jUtil.closeSafely(transaction, log)\n Neo4jUtil.closeSafely(session, log)\n driverCache.close()\n }\n\n def hasError(): Boolean = error\n\n protected def query(): String = {\n new Neo4jQueryService(\n options,\n new Neo4jQueryReadStrategy(\n neo4j,\n filters,\n partitionSkipLimit,\n requiredColumns.fieldNames,\n aggregateColumns,\n jobId\n )\n )\n .createQuery()\n }\n\n protected def queryParameters: util.Map[String, Any] = values\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/service/MappingService.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.service\n\nimport org.apache.spark.internal.Logging\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.driver.Record\nimport org.neo4j.driver.Value\nimport org.neo4j.driver.Values\nimport org.neo4j.driver.internal.value.MapValue\nimport org.neo4j.driver.types.Node\nimport org.neo4j.spark.converter.Neo4jToSparkDataConverter\nimport org.neo4j.spark.converter.SparkToNeo4jDataConverter\nimport org.neo4j.spark.service.Neo4jWriteMappingStrategy.KEYS\nimport org.neo4j.spark.service.Neo4jWriteMappingStrategy.PROPERTIES\nimport org.neo4j.spark.util.Neo4jImplicits._\nimport org.neo4j.spark.util.Neo4jNodeMetadata\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.Neo4jUtil\nimport org.neo4j.spark.util.QueryType\nimport org.neo4j.spark.util.RelationshipSaveStrategy\n\nimport java.util\nimport java.util.function\nimport java.util.function.BiConsumer\n\nimport scala.collection.JavaConverters._\nimport scala.collection.mutable\n\nclass Neo4jWriteMappingStrategy(private val options: Neo4jOptions)\n extends Neo4jMappingStrategy[InternalRow, Option[java.util.Map[String, AnyRef]]]\n with Logging {\n\n private val dataConverter = SparkToNeo4jDataConverter(options)\n\n override def node(row: InternalRow, schema: StructType): Option[java.util.Map[String, AnyRef]] = {\n val rowMap: java.util.Map[String, Object] = new java.util.HashMap[String, Object]\n val keys: java.util.Map[String, Object] = new java.util.HashMap[String, Object]\n val properties: java.util.Map[String, Object] = new java.util.HashMap[String, Object]\n rowMap.put(KEYS, keys)\n rowMap.put(PROPERTIES, properties)\n\n query(row, schema)\n .get\n .forEach(new BiConsumer[String, AnyRef] {\n override def accept(key: String, value: AnyRef): Unit = if (options.nodeMetadata.nodeKeys.contains(key)) {\n keys.put(options.nodeMetadata.nodeKeys.getOrElse(key, key), value)\n } else {\n properties.put(options.nodeMetadata.properties.getOrElse(key, key), value)\n }\n })\n\n if (options.nodeMetadata.skipNullKeys && containsNull(keys)) {\n logSkipping(\"node keys\", options.nodeMetadata.nodeKeys.values)\n None\n } else {\n Some(rowMap)\n }\n }\n\n private def nativeStrategyConsumer(): MappingBiConsumer = new MappingBiConsumer {\n\n override def accept(key: String, value: AnyRef): Unit = {\n if (key.startsWith(Neo4jUtil.RELATIONSHIP_ALIAS.concat(\".\"))) {\n relMap.get(PROPERTIES).put(key.removeAlias(), value)\n } else if (key.startsWith(Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS.concat(\".\"))) {\n if (options.relationshipMetadata.source.nodeKeys.contains(key)) {\n sourceNodeMap.get(KEYS).put(key.removeAlias(), value)\n } else {\n sourceNodeMap.get(PROPERTIES).put(key.removeAlias(), value)\n }\n } else if (key.startsWith(Neo4jUtil.RELATIONSHIP_TARGET_ALIAS.concat(\".\"))) {\n if (options.relationshipMetadata.target.nodeKeys.contains(key)) {\n targetNodeMap.get(KEYS).put(key.removeAlias(), value)\n } else {\n targetNodeMap.get(PROPERTIES).put(key.removeAlias(), value)\n }\n }\n }\n }\n\n private def addToNodeMap(\n nodeMap: util.Map[String, util.Map[String, AnyRef]],\n nodeMetadata: Neo4jNodeMetadata,\n key: String,\n value: AnyRef\n ): Unit = {\n if (nodeMetadata.nodeKeys.contains(key)) {\n nodeMap.get(KEYS).put(nodeMetadata.nodeKeys.getOrElse(key, key), value)\n }\n if (nodeMetadata.properties.contains(key)) {\n nodeMap.get(PROPERTIES).put(nodeMetadata.properties.getOrElse(key, key), value)\n }\n }\n\n private def keysStrategyConsumer(): MappingBiConsumer = new MappingBiConsumer {\n\n override def accept(key: String, value: AnyRef): Unit = {\n val source = options.relationshipMetadata.source\n val target = options.relationshipMetadata.target\n\n addToNodeMap(sourceNodeMap, source, key, value)\n addToNodeMap(targetNodeMap, target, key, value)\n\n if (options.relationshipMetadata.relationshipKeys.contains(key)) {\n relMap.get(KEYS).put(options.relationshipMetadata.relationshipKeys.getOrElse(key, key), value)\n } else {\n val propertyKey = options.relationshipMetadata.properties match {\n case Some(relProperties) => relProperties.get(key)\n case None =>\n if (!source.includesProperty(key) && !target.includesProperty(key)) {\n Some(key)\n } else {\n None\n }\n }\n propertyKey.foreach(k => relMap.get(PROPERTIES).put(k, value))\n }\n }\n }\n\n override def relationship(row: InternalRow, schema: StructType): Option[java.util.Map[String, AnyRef]] = {\n val rowMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]\n\n val consumer = options.relationshipMetadata.saveStrategy match {\n case RelationshipSaveStrategy.NATIVE => nativeStrategyConsumer()\n case RelationshipSaveStrategy.KEYS => keysStrategyConsumer()\n }\n\n query(row, schema).get.forEach(consumer)\n\n if (\n options.relationshipMetadata.saveStrategy.equals(RelationshipSaveStrategy.NATIVE)\n && consumer.relMap.get(PROPERTIES).isEmpty\n && consumer.sourceNodeMap.get(PROPERTIES).isEmpty && consumer.sourceNodeMap.get(KEYS).isEmpty\n && consumer.targetNodeMap.get(PROPERTIES).isEmpty && consumer.targetNodeMap.get(KEYS).isEmpty\n ) {\n throw new IllegalArgumentException(\n \"NATIVE write strategy requires a schema like: rel.[props], source.[props], target.[props]. \" +\n \"All of these columns are empty in the current schema.\"\n )\n }\n\n if (options.relationshipMetadata.skipNullKeys && containsNull(consumer.relMap, KEYS)) {\n logSkipping(\"relationship keys\", options.relationshipMetadata.relationshipKeys.values)\n None\n } else if (options.relationshipMetadata.source.skipNullKeys && containsNull(consumer.sourceNodeMap, KEYS)) {\n logSkipping(\"source node keys\", options.relationshipMetadata.source.nodeKeys.values)\n None\n } else if (options.relationshipMetadata.target.skipNullKeys && containsNull(consumer.targetNodeMap, KEYS)) {\n logSkipping(\"target node keys\", options.relationshipMetadata.target.nodeKeys.values)\n None\n } else {\n rowMap.put(Neo4jUtil.RELATIONSHIP_ALIAS, consumer.relMap)\n rowMap.put(Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS, consumer.sourceNodeMap)\n rowMap.put(Neo4jUtil.RELATIONSHIP_TARGET_ALIAS, consumer.targetNodeMap)\n Some(rowMap)\n }\n }\n\n override def query(row: InternalRow, schema: StructType): Option[java.util.Map[String, AnyRef]] = {\n val seq = row.toSeq(schema)\n Some(\n schema.indices\n .flatMap(i => {\n val field = schema(i)\n val neo4jValue = dataConverter.convert(seq(i), field.dataType)\n neo4jValue match {\n case map: MapValue =>\n map.asMap().asScala.toMap\n .flattenMap(field.name, options.schemaMetadata.mapGroupDuplicateKeys)\n .mapValues(value => Values.value(value).asInstanceOf[AnyRef])\n .toSeq\n case _ => Seq((field.name, neo4jValue))\n }\n })\n .toMap\n .asJava\n )\n }\n\n // Helper methods\n\n private def containsNull(map: java.util.Map[String, Object]): Boolean = {\n map.containsValue(Values.NULL)\n }\n\n private def containsNull(map: java.util.Map[String, java.util.Map[String, AnyRef]], key: String): Boolean = {\n map.get(key).containsValue(Values.NULL)\n }\n\n private def logSkipping(keyType: String, keys: Iterable[String]): Unit = {\n logTrace(s\"Skipping row because it contains null value for one of the $keyType: [${keys.mkString(\", \")}]\")\n }\n}\n\nclass Neo4jReadMappingStrategy(private val options: Neo4jOptions, requiredColumns: StructType)\n extends Neo4jMappingStrategy[Record, InternalRow] {\n\n private val dataConverter = Neo4jToSparkDataConverter(options)\n\n override def node(record: Record, schema: StructType): InternalRow = {\n if (requiredColumns.nonEmpty) {\n query(record, schema)\n } else {\n val node = record.get(Neo4jUtil.NODE_ALIAS).asNode()\n val nodeMap = new util.HashMap[String, Any](node.asMap())\n nodeMap.put(Neo4jUtil.INTERNAL_ID_FIELD, node.id())\n nodeMap.put(Neo4jUtil.INTERNAL_LABELS_FIELD, node.labels())\n\n mapToInternalRow(nodeMap, schema)\n }\n }\n\n private def mapToInternalRow(map: util.Map[String, Any], schema: StructType) = InternalRow\n .fromSeq(\n schema.map(field => dataConverter.convert(map.get(field.name), field.dataType))\n )\n\n private def flatRelNodeMapping(node: Node, alias: String): mutable.Map[String, Any] = {\n val nodeMap: mutable.Map[String, Any] = node.asMap().asScala\n .map(t => (s\"$alias.${t._1}\", t._2))\n nodeMap.put(\n s\"<$alias.${\n Neo4jUtil.INTERNAL_ID_FIELD\n .replaceAll(\"[<|>]\", \"\")\n }>\",\n node.id()\n )\n nodeMap.put(\n s\"<$alias.${\n Neo4jUtil.INTERNAL_LABELS_FIELD\n .replaceAll(\"[<|>]\", \"\")\n }>\",\n node.labels()\n )\n nodeMap\n }\n\n private def mapRelNodeMapping(node: Node, alias: String): Map[String, util.Map[String, String]] = {\n val nodeMap: util.Map[String, String] =\n new util.HashMap[String, String](node.asMap(new function.Function[Value, String] {\n override def apply(t: Value): String = t.toString\n }))\n\n nodeMap.put(Neo4jUtil.INTERNAL_ID_FIELD, Neo4jUtil.mapper.writeValueAsString(node.id()))\n nodeMap.put(Neo4jUtil.INTERNAL_LABELS_FIELD, Neo4jUtil.mapper.writeValueAsString(node.labels()))\n Map(s\"<$alias>\" -> nodeMap)\n }\n\n override def relationship(record: Record, schema: StructType): InternalRow = {\n if (requiredColumns.nonEmpty) {\n query(record, schema)\n } else {\n val rel = record.get(Neo4jUtil.RELATIONSHIP_ALIAS).asRelationship()\n val relMap = new util.HashMap[String, Any](rel.asMap())\n .asScala\n .map(t => (s\"rel.${t._1}\", t._2))\n .asJava\n relMap.put(Neo4jUtil.INTERNAL_REL_ID_FIELD, rel.id())\n relMap.put(Neo4jUtil.INTERNAL_REL_TYPE_FIELD, rel.`type`())\n\n val source = record.get(Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS).asNode()\n val target = record.get(Neo4jUtil.RELATIONSHIP_TARGET_ALIAS).asNode()\n\n val (sourceMap, targetMap) = if (options.relationshipMetadata.nodeMap) {\n (\n mapRelNodeMapping(source, Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS),\n mapRelNodeMapping(target, Neo4jUtil.RELATIONSHIP_TARGET_ALIAS)\n )\n } else {\n (\n flatRelNodeMapping(source, Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS),\n flatRelNodeMapping(target, Neo4jUtil.RELATIONSHIP_TARGET_ALIAS)\n )\n }\n\n relMap.putAll(sourceMap.toMap.asJava)\n relMap.putAll(targetMap.toMap.asJava)\n\n mapToInternalRow(relMap, schema)\n }\n }\n\n override def query(elem: Record, schema: StructType): InternalRow = mapToInternalRow(\n elem.asMap(new function.Function[Value, Any] {\n override def apply(t: Value): Any = t.asObject()\n }),\n schema\n )\n}\n\nabstract class Neo4jMappingStrategy[IN, OUT] extends Serializable {\n def node(elem: IN, schema: StructType): OUT\n\n def relationship(elem: IN, schema: StructType): OUT\n\n def query(elem: IN, schema: StructType): OUT\n}\n\nclass MappingService[IN, OUT](private val strategy: Neo4jMappingStrategy[IN, OUT], private val options: Neo4jOptions)\n extends Serializable {\n\n def convert(record: IN, schema: StructType): OUT = options.query.queryType match {\n case QueryType.LABELS => strategy.node(record, schema)\n case QueryType.RELATIONSHIP => strategy.relationship(record, schema)\n case QueryType.QUERY => strategy.query(record, schema)\n case QueryType.GDS => strategy.query(record, schema)\n }\n\n}\n\nobject Neo4jWriteMappingStrategy {\n val KEYS = \"keys\"\n val PROPERTIES = \"properties\"\n}\n\nabstract private class MappingBiConsumer extends BiConsumer[String, AnyRef] {\n\n val relMap = new util.HashMap[String, util.Map[String, AnyRef]]()\n val sourceNodeMap = new util.HashMap[String, util.Map[String, AnyRef]]()\n val targetNodeMap = new util.HashMap[String, util.Map[String, AnyRef]]()\n\n relMap.put(KEYS, new util.HashMap[String, AnyRef]())\n relMap.put(PROPERTIES, new util.HashMap[String, AnyRef]())\n sourceNodeMap.put(PROPERTIES, new util.HashMap[String, AnyRef]())\n sourceNodeMap.put(KEYS, new util.HashMap[String, AnyRef]())\n targetNodeMap.put(PROPERTIES, new util.HashMap[String, AnyRef]())\n targetNodeMap.put(KEYS, new util.HashMap[String, AnyRef]())\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/service/Neo4jQueryService.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.service\n\nimport org.apache.commons.lang3.StringUtils\nimport org.apache.spark.internal.Logging\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.connector.expressions.SortDirection\nimport org.apache.spark.sql.connector.expressions.SortOrder\nimport org.apache.spark.sql.connector.expressions.aggregate._\nimport org.apache.spark.sql.sources.And\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.sources.Or\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.cypherdsl.core._\nimport org.neo4j.cypherdsl.core.renderer.Renderer\nimport org.neo4j.spark.cypher.Cypher5Renderer\nimport org.neo4j.spark.cypher.CypherVersionSelector.selectCypherVersionClause\nimport org.neo4j.spark.util.Neo4jImplicits._\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.Neo4jUtil\nimport org.neo4j.spark.util.NodeSaveMode\nimport org.neo4j.spark.util.QueryType\n\nimport scala.collection.JavaConverters._\n\nclass Neo4jQueryWriteStrategy(private val neo4j: Neo4j, private val saveMode: SaveMode) extends Neo4jQueryStrategy {\n\n override def createStatementForQuery(options: Neo4jOptions): String =\n s\"\"\"WITH ${\"$\"}scriptResult AS ${Neo4jQueryStrategy.VARIABLE_SCRIPT_RESULT}\n |UNWIND ${\"$\"}events AS ${Neo4jQueryStrategy.VARIABLE_EVENT}\n |${options.query.value}\n |\"\"\".stripMargin\n\n private def createPropsList(props: Map[String, String], prefix: String): String = {\n props\n .map(key => {\n s\"${key._2.quote()}: ${Neo4jQueryStrategy.VARIABLE_EVENT}.$prefix.${key._2.quote()}\"\n }).mkString(\", \")\n }\n\n private def keywordFromSaveMode(saveMode: Any): String = {\n saveMode match {\n case NodeSaveMode.Overwrite | SaveMode.Overwrite => \"MERGE\"\n case NodeSaveMode.ErrorIfExists | SaveMode.ErrorIfExists | SaveMode.Append | NodeSaveMode.Append => \"CREATE\"\n case NodeSaveMode.Match => \"MATCH\"\n case _ => throw new UnsupportedOperationException(s\"SaveMode $saveMode not supported\")\n }\n }\n\n private def createQueryPart(keyword: String, labels: String, keys: String, alias: String): String = {\n val setStatement = if (!keyword.equals(\"MATCH\"))\n s\" SET $alias += ${Neo4jQueryStrategy.VARIABLE_EVENT}.$alias.${Neo4jWriteMappingStrategy.PROPERTIES}\"\n else \"\"\n s\"\"\"$keyword ($alias${if (labels.isEmpty) \"\" else s\":$labels\"} ${\n if (keys.isEmpty) \"\"\n else s\"{$keys}\"\n })$setStatement\"\"\".stripMargin\n }\n\n override def createStatementForRelationships(options: Neo4jOptions): String = {\n val relationshipKeyword = keywordFromSaveMode(saveMode)\n val sourceKeyword = keywordFromSaveMode(options.relationshipMetadata.sourceSaveMode)\n val targetKeyword = keywordFromSaveMode(options.relationshipMetadata.targetSaveMode)\n\n val relationship = options.relationshipMetadata.relationshipType.quote()\n\n val sourceLabels = options.relationshipMetadata.source.labels\n .map(_.quote())\n .mkString(\":\")\n\n val targetLabels = options.relationshipMetadata.target.labels\n .map(_.quote())\n .mkString(\":\")\n\n val sourceKeys = createPropsList(\n options.relationshipMetadata.source.nodeKeys,\n s\"source.${Neo4jWriteMappingStrategy.KEYS}\"\n )\n val targetKeys = createPropsList(\n options.relationshipMetadata.target.nodeKeys,\n s\"target.${Neo4jWriteMappingStrategy.KEYS}\"\n )\n val sourceQueryPart = createQueryPart(sourceKeyword, sourceLabels, sourceKeys, Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS)\n val targetQueryPart = createQueryPart(targetKeyword, targetLabels, targetKeys, Neo4jUtil.RELATIONSHIP_TARGET_ALIAS)\n\n val withQueryPart = if (sourceKeyword != \"MATCH\" && targetKeyword == \"MATCH\")\n \"\\nWITH source, event\"\n else {\n \"\"\n }\n\n val relKeys = if (options.relationshipMetadata.relationshipKeys.nonEmpty) {\n options.relationshipMetadata.relationshipKeys\n .map(t =>\n s\"${t._2}: ${Neo4jQueryStrategy.VARIABLE_EVENT}.${Neo4jUtil.RELATIONSHIP_ALIAS}.${Neo4jWriteMappingStrategy.KEYS}.${t._1}\"\n )\n .mkString(\"{\", \", \", \"}\")\n } else {\n \"\"\n }\n\n s\"\"\"${selectCypherVersionClause(neo4j)}UNWIND ${\"$\"}events AS ${Neo4jQueryStrategy.VARIABLE_EVENT}\n |$sourceQueryPart$withQueryPart\n |$targetQueryPart\n |$relationshipKeyword (${Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS})-[${Neo4jUtil.RELATIONSHIP_ALIAS}:$relationship$relKeys]->(${Neo4jUtil.RELATIONSHIP_TARGET_ALIAS})\n |SET ${Neo4jUtil.RELATIONSHIP_ALIAS} += ${Neo4jQueryStrategy.VARIABLE_EVENT}.${Neo4jUtil.RELATIONSHIP_ALIAS}.${Neo4jWriteMappingStrategy.PROPERTIES}\n |\"\"\".stripMargin\n }\n\n override def createStatementForNodes(options: Neo4jOptions): String = {\n val keyword = keywordFromSaveMode(saveMode)\n\n val labels = options.nodeMetadata.labels\n .map(_.quote())\n .mkString(\":\")\n\n val keys = createPropsList(\n options.nodeMetadata.nodeKeys,\n Neo4jWriteMappingStrategy.KEYS\n )\n\n s\"\"\"${selectCypherVersionClause(neo4j)}UNWIND ${\"$\"}events AS ${Neo4jQueryStrategy.VARIABLE_EVENT}\n |$keyword (node${if (labels.isEmpty) \"\" else s\":$labels\"} ${if (keys.isEmpty) \"\" else s\"{$keys}\"})\n |SET node += ${Neo4jQueryStrategy.VARIABLE_EVENT}.${Neo4jWriteMappingStrategy.PROPERTIES}\n |\"\"\".stripMargin\n }\n\n override def createStatementForGDS(options: Neo4jOptions): String =\n throw new UnsupportedOperationException(\"Write operations with GDS are currently not supported\")\n}\n\nclass Neo4jQueryReadStrategy(\n neo4j: Neo4j,\n filters: Array[Filter] = Array.empty[Filter],\n partitionPagination: PartitionPagination = PartitionPagination.EMPTY,\n requiredColumns: Seq[String] = Seq.empty,\n aggregateColumns: Array[AggregateFunc] = Array.empty,\n jobId: String = \"\"\n) extends Neo4jQueryStrategy with Logging {\n private val renderer: Renderer = new Cypher5Renderer(neo4j)\n\n private val hasSkipLimit: Boolean = partitionPagination.skip != -1 && partitionPagination.topN.limit != -1\n\n override def createStatementForQuery(options: Neo4jOptions): String = {\n if (partitionPagination.topN.orders.nonEmpty) {\n logWarning(\n s\"\"\"Top N push-down optimizations with aggregations are not supported for custom queries.\n |\\tThese aggregations are going to be ignored.\n |\\tPlease specify the aggregations in the custom query directly\"\"\".stripMargin\n )\n }\n\n val limitedQuery = if (hasSkipLimit) {\n s\"${options.query.value} SKIP ${partitionPagination.skip} LIMIT ${partitionPagination.topN.limit}\"\n } else {\n s\"${options.query.value}\"\n }\n\n s\"WITH $$scriptResult AS scriptResult $limitedQuery\"\n }\n\n override def createStatementForRelationships(options: Neo4jOptions): String = {\n val sourceNode = createNode(Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS, options.relationshipMetadata.source.labels)\n val targetNode = createNode(Neo4jUtil.RELATIONSHIP_TARGET_ALIAS, options.relationshipMetadata.target.labels)\n\n val relationship = sourceNode.relationshipTo(targetNode, options.relationshipMetadata.relationshipType)\n .named(Neo4jUtil.RELATIONSHIP_ALIAS)\n\n val matchQuery: StatementBuilder.OngoingReadingWithoutWhere =\n filterRelationship(sourceNode, targetNode, relationship)\n val returnExpressions: Seq[Expression] = buildReturnExpression(sourceNode, targetNode, relationship)\n val stmt = if (aggregateColumns.isEmpty) {\n val query = matchQuery.returning(returnExpressions: _*)\n buildStatement(options, query, relationship)\n } else {\n buildStatementAggregation(options, matchQuery, relationship, returnExpressions)\n }\n renderer.render(stmt)\n }\n\n private def convertSort(entity: PropertyContainer, order: SortOrder): SortItem = {\n val sortExpression = order.expression().describe()\n\n val container: Option[PropertyContainer] = entity match {\n case relationship: Relationship =>\n if (sortExpression.contains(s\"${Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS}.\")) {\n Some(relationship.getLeft)\n } else if (sortExpression.contains(s\"${Neo4jUtil.RELATIONSHIP_TARGET_ALIAS}.\")) {\n Some(relationship.getRight)\n } else if (sortExpression.contains(s\"${Neo4jUtil.RELATIONSHIP_ALIAS}.\")) {\n Some(relationship)\n } else {\n None\n }\n case _ => Some(entity)\n }\n val direction =\n if (order.direction() == SortDirection.ASCENDING) SortItem.Direction.ASC else SortItem.Direction.DESC\n\n Cypher.sort(\n container\n .map(_.property(sortExpression.removeAlias()))\n .getOrElse(Cypher.name(sortExpression.unquote())),\n direction\n )\n }\n\n private def buildReturnExpression(sourceNode: Node, targetNode: Node, relationship: Relationship): Seq[Expression] = {\n if (requiredColumns.isEmpty) {\n Seq(\n relationship.getRequiredSymbolicName,\n sourceNode.as(Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS),\n targetNode.as(Neo4jUtil.RELATIONSHIP_TARGET_ALIAS)\n )\n } else {\n requiredColumns.map(column => {\n val splatColumn = column.split('.')\n val entityName = splatColumn.head\n\n val entity = if (entityName.contains(Neo4jUtil.RELATIONSHIP_ALIAS)) {\n relationship\n } else if (entityName.contains(Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS)) {\n sourceNode\n } else if (entityName.contains(Neo4jUtil.RELATIONSHIP_TARGET_ALIAS)) {\n targetNode\n } else {\n null\n }\n\n if (entity != null && splatColumn.length == 1) {\n entity match {\n case n: Node => n.as(entityName.quote())\n case r: Relationship => r.getRequiredSymbolicName\n }\n } else {\n getCorrectProperty(column, entity)\n }\n })\n }\n }\n\n private def buildStatementAggregation(\n options: Neo4jOptions,\n query: StatementBuilder.OngoingReadingWithoutWhere,\n entity: PropertyContainer,\n fields: Seq[Expression]\n ): Statement = {\n val ret = if (hasSkipLimit) {\n val id = entity match {\n case node: Node => Functions.id(node)\n case rel: Relationship => Functions.id(rel)\n }\n query\n .`with`(entity)\n // Spark does not push down limits/top N when aggregation is involved\n .orderBy(id)\n .skip(partitionPagination.skip)\n .limit(partitionPagination.topN.limit)\n .returning(fields: _*)\n } else {\n val orderByProp = options.streamingOrderBy\n if (StringUtils.isBlank(orderByProp)) {\n query.returning(fields: _*)\n } else {\n query\n .`with`(entity)\n .orderBy(entity.property(orderByProp))\n .ascending()\n .returning(fields: _*)\n }\n }\n ret.build()\n }\n\n private def buildStatement(\n options: Neo4jOptions,\n returning: StatementBuilder.TerminalExposesSkip\n with StatementBuilder.TerminalExposesLimit\n with StatementBuilder.TerminalExposesOrderBy\n with StatementBuilder.BuildableStatement[_],\n entity: PropertyContainer = null\n ): Statement = {\n\n def addSkipLimit(ret: StatementBuilder.TerminalExposesSkip\n with StatementBuilder.TerminalExposesLimit\n with StatementBuilder.BuildableStatement[_]) = {\n\n if (partitionPagination.skip == 0) {\n ret.limit(partitionPagination.topN.limit)\n } else {\n ret.skip(partitionPagination.skip)\n .limit(partitionPagination.topN.limit)\n }\n }\n\n val ret = if (entity == null) {\n if (hasSkipLimit) addSkipLimit(returning) else returning\n } else {\n if (hasSkipLimit) {\n if (options.partitions == 1 || partitionPagination.topN.orders.nonEmpty) {\n addSkipLimit(returning.orderBy(partitionPagination.topN.orders.map(order => convertSort(entity, order)): _*))\n } else {\n val id = entity match {\n case node: Node => Functions.id(node)\n case rel: Relationship => Functions.id(rel)\n }\n addSkipLimit(returning.orderBy(id))\n }\n } else {\n val orderByProp = options.streamingOrderBy\n if (StringUtils.isBlank(orderByProp)) returning else returning.orderBy(entity.property(orderByProp))\n }\n }\n ret.build()\n }\n\n private def filterRelationship(sourceNode: Node, targetNode: Node, relationship: Relationship) = {\n val matchQuery = Cypher.`match`(sourceNode).`match`(targetNode).`match`(relationship)\n\n def getContainer(filter: Filter): PropertyContainer = {\n if (filter.isAttribute(Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS)) {\n sourceNode\n } else if (filter.isAttribute(Neo4jUtil.RELATIONSHIP_TARGET_ALIAS)) {\n targetNode\n } else if (filter.isAttribute(Neo4jUtil.RELATIONSHIP_ALIAS)) {\n relationship\n } else {\n throw new IllegalArgumentException(s\"Attribute '${filter.getAttribute.get}' is not valid\")\n }\n }\n\n if (filters.nonEmpty) {\n def mapFilter(filter: Filter): Condition = {\n filter match {\n case and: And => mapFilter(and.left).and(mapFilter(and.right))\n case or: Or => mapFilter(or.left).or(mapFilter(or.right))\n case filter: Filter =>\n Neo4jUtil.mapSparkFiltersToCypher(filter, getContainer(filter), filter.getAttributeWithoutEntityName)\n }\n }\n\n val cypherFilters = filters.map(mapFilter)\n\n assembleConditionQuery(matchQuery, cypherFilters)\n }\n matchQuery\n }\n\n private def getCorrectProperty(column: String, entity: PropertyContainer): Expression = {\n def propertyOrSymbolicName(col: String) = {\n if (entity != null) entity.property(col) else Cypher.name(col)\n }\n\n column match {\n case Neo4jUtil.INTERNAL_ID_FIELD => Functions.id(entity.asInstanceOf[Node]).as(Neo4jUtil.INTERNAL_ID_FIELD)\n case Neo4jUtil.INTERNAL_REL_ID_FIELD =>\n Functions.id(entity.asInstanceOf[Relationship]).as(Neo4jUtil.INTERNAL_REL_ID_FIELD)\n case Neo4jUtil.INTERNAL_REL_SOURCE_ID_FIELD =>\n Functions.id(entity.asInstanceOf[Node]).as(Neo4jUtil.INTERNAL_REL_SOURCE_ID_FIELD)\n case Neo4jUtil.INTERNAL_REL_TARGET_ID_FIELD =>\n Functions.id(entity.asInstanceOf[Node]).as(Neo4jUtil.INTERNAL_REL_TARGET_ID_FIELD)\n case Neo4jUtil.INTERNAL_REL_TYPE_FIELD =>\n Functions.`type`(entity.asInstanceOf[Relationship]).as(Neo4jUtil.INTERNAL_REL_TYPE_FIELD)\n case Neo4jUtil.INTERNAL_LABELS_FIELD =>\n Functions.labels(entity.asInstanceOf[Node]).as(Neo4jUtil.INTERNAL_LABELS_FIELD)\n case Neo4jUtil.INTERNAL_REL_SOURCE_LABELS_FIELD =>\n Functions.labels(entity.asInstanceOf[Node]).as(Neo4jUtil.INTERNAL_REL_SOURCE_LABELS_FIELD)\n case Neo4jUtil.INTERNAL_REL_TARGET_LABELS_FIELD =>\n Functions.labels(entity.asInstanceOf[Node]).as(Neo4jUtil.INTERNAL_REL_TARGET_LABELS_FIELD)\n case \"*\" => Asterisk.INSTANCE\n case name => {\n val cleanedName = name.removeAlias()\n aggregateColumns.find(_.toString == name)\n .map {\n case count: Count => {\n val col = count.column().describe().unquote().removeAlias()\n val prop = propertyOrSymbolicName(col)\n if (count.isDistinct) {\n Functions.countDistinct(prop).as(name)\n } else {\n Functions.count(prop).as(name)\n }\n }\n case countStar: CountStar => Functions.count(Asterisk.INSTANCE).as(name)\n case max: Max =>\n val col = max.column().describe().unquote().removeAlias()\n val prop = propertyOrSymbolicName(col)\n Functions.max(prop).as(name)\n case min: Min =>\n val col = min.column().describe().unquote().removeAlias()\n val prop = propertyOrSymbolicName(col)\n Functions.min(prop).as(name)\n case sum: Sum => {\n val col = sum.column().describe().unquote().removeAlias()\n val prop = propertyOrSymbolicName(col)\n if (sum.isDistinct) {\n Functions.sumDistinct(prop).as(name)\n } else {\n Functions.sum(prop).as(name)\n }\n }\n }\n .getOrElse(propertyOrSymbolicName(cleanedName).as(name))\n .asInstanceOf[Expression]\n }\n }\n }\n\n override def createStatementForNodes(options: Neo4jOptions): String = {\n val node = createNode(Neo4jUtil.NODE_ALIAS, options.nodeMetadata.labels)\n val matchQuery = filterNode(node)\n val expressions = requiredColumns.map(column => getCorrectProperty(column, node))\n val stmt = if (aggregateColumns.nonEmpty) {\n buildStatementAggregation(options, matchQuery, node, expressions)\n } else {\n val ret = if (requiredColumns.isEmpty) {\n matchQuery.returning(node)\n } else {\n matchQuery.returning(expressions: _*)\n }\n buildStatement(options, ret, node)\n }\n renderer.render(stmt)\n }\n\n private def filterNode(node: Node) = {\n val matchQuery = Cypher.`match`(node)\n\n if (filters.nonEmpty) {\n def mapFilter(filter: Filter): Condition = {\n filter match {\n case and: And => mapFilter(and.left).and(mapFilter(and.right))\n case or: Or => mapFilter(or.left).or(mapFilter(or.right))\n case filter: Filter => Neo4jUtil.mapSparkFiltersToCypher(filter, node)\n }\n }\n\n val cypherFilters = filters.map(mapFilter)\n assembleConditionQuery(matchQuery, cypherFilters)\n }\n matchQuery\n }\n\n def createStatementForNodeCount(options: Neo4jOptions): String = {\n val node = createNode(Neo4jUtil.NODE_ALIAS, options.nodeMetadata.labels)\n val matchQuery = filterNode(node)\n renderer.render(buildStatement(options, matchQuery.returning(Functions.count(node).as(\"count\"))))\n }\n\n def createStatementForRelationshipCount(options: Neo4jOptions): String = {\n val sourceNode = createNode(Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS, options.relationshipMetadata.source.labels)\n val targetNode = createNode(Neo4jUtil.RELATIONSHIP_TARGET_ALIAS, options.relationshipMetadata.target.labels)\n\n val relationship = sourceNode.relationshipTo(targetNode, options.relationshipMetadata.relationshipType)\n .named(Neo4jUtil.RELATIONSHIP_ALIAS)\n\n val matchQuery: StatementBuilder.OngoingReadingWithoutWhere =\n filterRelationship(sourceNode, targetNode, relationship)\n\n renderer.render(buildStatement(options, matchQuery.returning(Functions.count(sourceNode).as(\"count\"))))\n }\n\n private def assembleConditionQuery(\n matchQuery: StatementBuilder.OngoingReadingWithoutWhere,\n filters: Array[Condition]\n ): StatementBuilder.OngoingReadingWithWhere = {\n matchQuery.where(\n filters.fold(Conditions.noCondition()) { (a, b) => a.and(b) }\n )\n }\n\n private def createNode(name: String, labels: Seq[String]) = {\n val primaryLabel = labels.head\n val otherLabels = labels.tail\n if (labels.isEmpty) {\n Cypher.anyNode(name)\n } else {\n Cypher.node(primaryLabel, otherLabels.asJava).named(name)\n }\n }\n\n override def createStatementForGDS(options: Neo4jOptions): String = {\n val retCols = requiredColumns.map(column => getCorrectProperty(column, null))\n // we need it in order to parse the field YIELD by the GDS procedure...\n val (yieldFields, args) = Neo4jUtil.callSchemaService(\n neo4j,\n options,\n jobId,\n filters,\n { ss => (ss.struct().fieldNames, ss.inputForGDSProc(options.query.value)) }\n )\n\n val cypherParams = args\n .filter(t => {\n if (!t._2) {\n true\n } else {\n options.gdsMetadata.parameters.containsKey(t._1)\n }\n })\n .map(_._1)\n .map(Cypher.parameter)\n val statement = Cypher.call(options.query.value)\n .withArgs(cypherParams: _*)\n .`yield`(yieldFields: _*)\n .returning(retCols: _*)\n .build()\n renderer.render(statement)\n }\n}\n\nobject Neo4jQueryStrategy {\n val VARIABLE_EVENT = \"event\"\n val VARIABLE_EVENTS = \"events\"\n val VARIABLE_SCRIPT_RESULT = \"scriptResult\"\n val VARIABLE_STREAM = \"stream\"\n}\n\nabstract class Neo4jQueryStrategy {\n\n def createStatementForQuery(options: Neo4jOptions): String\n\n def createStatementForRelationships(options: Neo4jOptions): String\n\n def createStatementForNodes(options: Neo4jOptions): String\n\n def createStatementForGDS(options: Neo4jOptions): String\n}\n\nclass Neo4jQueryService(private val options: Neo4jOptions, val strategy: Neo4jQueryStrategy) extends Serializable {\n\n def createQuery(): String = options.query.queryType match {\n case QueryType.LABELS => strategy.createStatementForNodes(options)\n case QueryType.RELATIONSHIP => strategy.createStatementForRelationships(options)\n case QueryType.QUERY => strategy.createStatementForQuery(options)\n case QueryType.GDS => strategy.createStatementForGDS(options)\n case _ => throw new UnsupportedOperationException(\n s\"\"\"Query Type not supported.\n |You provided ${options.query.queryType},\n |supported types: ${QueryType.values.mkString(\",\")}\"\"\".stripMargin\n )\n }\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/service/SchemaService.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.service\n\nimport org.apache.commons.lang3.exception.ExceptionUtils\nimport org.apache.spark.internal.Logging\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.types.DataTypes\nimport org.apache.spark.sql.types.StructField\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.driver.Record\nimport org.neo4j.driver.Session\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.Value\nimport org.neo4j.driver.Values\nimport org.neo4j.driver.exceptions.ClientException\nimport org.neo4j.driver.summary\nimport org.neo4j.spark.config.TopN\nimport org.neo4j.spark.converter.CypherToSparkTypeConverter\nimport org.neo4j.spark.converter.SparkToCypherTypeConverter\nimport org.neo4j.spark.cypher.CypherVersionSelector.selectCypherVersionClause\nimport org.neo4j.spark.service.SchemaService.normalizedClassName\nimport org.neo4j.spark.service.SchemaService.normalizedClassNameFromGraphEntity\nimport org.neo4j.spark.util.Neo4jImplicits.CypherImplicits\nimport org.neo4j.spark.util.Neo4jImplicits.ValueImplicits\nimport org.neo4j.spark.util.OptimizationType\nimport org.neo4j.spark.util._\n\nimport java.util\nimport java.util.Collections\nimport java.util.function\n\nimport scala.collection.JavaConverters._\nimport scala.collection.mutable\nimport scala.collection.mutable.ArrayBuffer\n\nobject PartitionPagination {\n val EMPTY: PartitionPagination = PartitionPagination(0, -1, TopN(-1))\n}\n\ncase class PartitionPagination(partitionNumber: Int, skip: Long, topN: TopN)\n\nclass SchemaService(\n private val neo4j: Neo4j,\n private val options: Neo4jOptions,\n private val driverCache: DriverCache,\n private val filters: Array[Filter] = Array.empty\n) extends AutoCloseable with Logging {\n\n private val queryReadStrategy = new Neo4jQueryReadStrategy(neo4j, filters)\n\n private val session: Session = driverCache.getOrCreate().session(options.session.toNeo4jSession())\n\n private val sessionTransactionConfig = options.toNeo4jTransactionConfig\n\n private val cypherToSparkTypeConverter = CypherToSparkTypeConverter(options)\n\n private val sparkToCypherTypeConverter = SparkToCypherTypeConverter(options)\n\n private def structForNode(labels: Seq[String] = options.nodeMetadata.labels) = {\n val structFields: mutable.Buffer[StructField] = (try {\n val query =\n s\"\"\"${selectCypherVersionClause(neo4j)}CALL apoc.meta.nodeTypeProperties($$config)\n |YIELD propertyName, propertyTypes\n |WITH DISTINCT propertyName, propertyTypes\n |WITH propertyName, collect(propertyTypes) AS propertyTypes\n |RETURN propertyName, reduce(acc = [], elem IN propertyTypes | acc + elem) AS propertyTypes\n |\"\"\".stripMargin\n val apocConfig = options.apocConfig.procedureConfigMap\n .getOrElse(\"apoc.meta.nodeTypeProperties\", Map.empty[String, AnyRef])\n .asInstanceOf[Map[String, AnyRef]] ++ Map[String, AnyRef](\"includeLabels\" -> labels.asJava)\n retrieveSchemaFromApoc(query, Collections.singletonMap(\"config\", apocConfig.asJava))\n } catch {\n case e: ClientException =>\n logResolutionChange(\"Switching to query schema resolution\", e)\n // TODO get back to Cypher DSL when rand function will be available\n val query =\n s\"\"\"${selectCypherVersionClause(neo4j)}MATCH (${Neo4jUtil.NODE_ALIAS}:${labels.map(_.quote()).mkString(\":\")})\n |RETURN ${Neo4jUtil.NODE_ALIAS}\n |ORDER BY rand()\n |LIMIT ${options.schemaMetadata.flattenLimit}\n |\"\"\".stripMargin\n val params = Collections.emptyMap[String, AnyRef]()\n retrieveSchema(query, params, { record => record.get(Neo4jUtil.NODE_ALIAS).asNode.asMap.asScala.toMap })\n })\n .sortBy(t => t.name)\n\n structFields += StructField(\n Neo4jUtil.INTERNAL_LABELS_FIELD,\n DataTypes.createArrayType(DataTypes.StringType),\n nullable = true\n )\n structFields += StructField(Neo4jUtil.INTERNAL_ID_FIELD, DataTypes.LongType, nullable = false)\n StructType(structFields.reverse.toSeq)\n }\n\n private def retrieveSchemaFromApoc(\n query: String,\n params: java.util.Map[String, AnyRef]\n ): mutable.Buffer[StructField] = {\n val fields = session.run(query, params, sessionTransactionConfig)\n .list\n .asScala\n .filter(record => !record.get(\"propertyName\").isNull && !record.get(\"propertyName\").isEmpty)\n .map(record => {\n val fieldTypesList = record.get(\"propertyTypes\")\n .asList(new function.Function[Value, String]() {\n override def apply(v: Value): String = v.asString()\n })\n .asScala\n val fieldType: String = if (fieldTypesList.size > 1) {\n log.warn(\n s\"\"\"\n |The field ${record.get(\"propertyName\")} has different types: $fieldTypesList\n |Every value will be casted to string.\n |\"\"\".stripMargin\n )\n \"String\"\n } else {\n fieldTypesList.head\n }\n\n StructField(record.get(\"propertyName\").asString, cypherToSparkTypeConverter.convert(fieldType))\n })\n if (fields.isEmpty) {\n throw new ClientException(\"Unable to compute the resulting schema from APOC\")\n }\n fields\n }\n\n private def retrieveSchema(\n query: String,\n params: java.util.Map[String, AnyRef],\n extractFunction: Record => Map[String, AnyRef]\n ): mutable.Buffer[StructField] = {\n session.run(query, params, sessionTransactionConfig).list.asScala\n .flatMap(extractFunction)\n .groupBy(_._1)\n .mapValues(_.map(_._2))\n .map(t =>\n options.schemaMetadata.strategy match {\n case SchemaStrategy.SAMPLE => {\n val types = t._2.map(value => {\n if (options.query.queryType == QueryType.QUERY) {\n normalizedClassName(value, options)\n } else {\n normalizedClassNameFromGraphEntity(value, options)\n }\n }).toSet\n\n if (types.size > 1) {\n log.warn(\n s\"\"\"\n |The field ${t._1} has different types: ${types.toString}\n |Every value will be casted to string.\n |\"\"\".stripMargin\n )\n StructField(t._1, DataTypes.StringType)\n } else {\n val value = t._2.head\n StructField(t._1, cypherToSparkTypeConverter.convert(types.head, value))\n }\n }\n case SchemaStrategy.STRING => StructField(t._1, DataTypes.StringType)\n }\n )\n .toBuffer\n }\n\n private def mapStructField(alias: String, field: StructField): StructField = {\n val name = field.name match {\n case Neo4jUtil.INTERNAL_ID_FIELD | Neo4jUtil.INTERNAL_LABELS_FIELD =>\n s\"<$alias.${field.name.replaceAll(\"[<|>]\", \"\")}>\"\n case _ => s\"$alias.${field.name}\"\n }\n StructField(name, field.dataType, field.nullable, field.metadata)\n }\n\n private def structForRelationship() = {\n val structFields: mutable.Buffer[StructField] = ArrayBuffer(\n StructField(Neo4jUtil.INTERNAL_REL_ID_FIELD, DataTypes.LongType, false),\n StructField(Neo4jUtil.INTERNAL_REL_TYPE_FIELD, DataTypes.StringType, false)\n )\n\n if (options.relationshipMetadata.nodeMap) {\n structFields += StructField(\n s\"<${Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS}>\",\n DataTypes.createMapType(DataTypes.StringType, DataTypes.StringType),\n false\n )\n structFields += StructField(\n s\"<${Neo4jUtil.RELATIONSHIP_TARGET_ALIAS}>\",\n DataTypes.createMapType(DataTypes.StringType, DataTypes.StringType),\n false\n )\n } else {\n structFields ++= structForNode(options.relationshipMetadata.source.labels)\n .map(field => mapStructField(Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS, field))\n structFields ++= structForNode(options.relationshipMetadata.target.labels)\n .map(field => mapStructField(Neo4jUtil.RELATIONSHIP_TARGET_ALIAS, field))\n }\n\n structFields ++= (try {\n val query =\n s\"\"\"${selectCypherVersionClause(\n neo4j\n )}CALL apoc.meta.relTypeProperties($$config) YIELD sourceNodeLabels, targetNodeLabels,\n | propertyName, propertyTypes\n |WITH *\n |WHERE sourceNodeLabels = $$sourceLabels AND targetNodeLabels = $$targetLabels\n |RETURN *\n |\"\"\".stripMargin\n val apocConfig = options.apocConfig.procedureConfigMap\n .getOrElse(\"apoc.meta.relTypeProperties\", Map.empty[String, AnyRef])\n .asInstanceOf[Map[String, AnyRef]]\n val config = apocConfig ++ Map(\"includeRels\" -> Seq(options.relationshipMetadata.relationshipType).asJava)\n val params = Map[String, AnyRef](\n \"config\" -> config.asJava,\n \"sourceLabels\" -> options.relationshipMetadata.source.labels.asJava,\n \"targetLabels\" -> options.relationshipMetadata.target.labels.asJava\n )\n .asJava\n retrieveSchemaFromApoc(query, params)\n } catch {\n case e: ClientException =>\n logResolutionChange(\"Switching to query schema resolution\", e)\n // TODO get back to Cypher DSL when rand function will be available\n val query =\n s\"\"\"${selectCypherVersionClause(\n neo4j\n )}MATCH (${Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS}:${options.relationshipMetadata.source.labels.map(\n _.quote()\n ).mkString(\":\")})\n |MATCH (${Neo4jUtil.RELATIONSHIP_TARGET_ALIAS}:${options.relationshipMetadata.target.labels.map(\n _.quote()\n ).mkString(\":\")})\n |MATCH (${Neo4jUtil.RELATIONSHIP_SOURCE_ALIAS})-[${Neo4jUtil.RELATIONSHIP_ALIAS}:${options.relationshipMetadata.relationshipType}]->(${Neo4jUtil.RELATIONSHIP_TARGET_ALIAS})\n |RETURN ${Neo4jUtil.RELATIONSHIP_ALIAS}\n |ORDER BY rand()\n |LIMIT ${options.schemaMetadata.flattenLimit}\n |\"\"\".stripMargin\n val params = Collections.emptyMap[String, AnyRef]()\n retrieveSchema(\n query,\n params,\n { record => record.get(Neo4jUtil.RELATIONSHIP_ALIAS).asRelationship.asMap.asScala.toMap }\n )\n })\n .map(field => StructField(s\"rel.${field.name}\", field.dataType, field.nullable, field.metadata))\n .sortBy(t => t.name)\n StructType(structFields.toSeq)\n }\n\n private def structForQuery(): StructType = {\n val query = queryReadStrategy.createStatementForQuery(options)\n if (!isValidQuery(query, summary.QueryType.READ_ONLY)) {\n return new StructType()\n }\n\n val params = Map[String, AnyRef](\n Neo4jQueryStrategy.VARIABLE_SCRIPT_RESULT -> Collections.emptyList(),\n Neo4jQueryStrategy.VARIABLE_STREAM -> Collections.emptyMap()\n )\n .asJava\n\n val randLimitedQueryForSchema =\n s\"\"\"\n |$query\n |ORDER BY rand()\n |LIMIT ${options.schemaMetadata.flattenLimit}\n |\"\"\".stripMargin\n val randCallLimitedQueryForSchema =\n s\"\"\"\n |CALL {\n | $query\n |} RETURN *\n |ORDER BY rand()\n |LIMIT ${options.schemaMetadata.flattenLimit}\n |\"\"\".stripMargin\n\n val limitedQuery =\n if (isValidQuery(randLimitedQueryForSchema)) randLimitedQueryForSchema else randCallLimitedQueryForSchema\n\n val structFields = retrieveSchema(limitedQuery, params, { record => record.asMap.asScala.toMap })\n\n val columns = getReturnedColumns(query)\n if (columns.isEmpty && structFields.isEmpty) {\n throw new ClientException(\n \"Unable to compute the resulting schema; this may mean your result set is empty or your version of Neo4j does not permit schema inference for empty sets\"\n )\n }\n\n if (columns.isEmpty) {\n return StructType(structFields.toSeq)\n }\n\n val sortedStructFields = if (structFields.isEmpty) {\n // df: we arrived here because there are no data returned by the query\n // so we want to return an empty dataset which schema is equals to the columns\n // specified by the RETURN statement\n columns.map(StructField(_, DataTypes.StringType))\n } else {\n try {\n columns.map(column => structFields.find(_.name.quote() == column.quote()).orNull).filter(_ != null)\n } catch {\n case _: Throwable => structFields.toArray\n }\n }\n\n StructType(sortedStructFields)\n }\n\n private def structForGDS() = {\n val query =\n s\"\"\"\n |${selectCypherVersionClause(neo4j)}CALL gds.list() YIELD name, signature, type\n |WHERE name = $$procName AND type = 'procedure'\n |WITH split(signature, ') :: (')[1] AS fields\n |WITH substring(fields, 0, size(fields) - 1) AS fields\n |WITH split(fields, ',') AS fields\n |WITH [field IN fields | split(field, ' :: ')] AS fields\n |UNWIND fields AS field\n |WITH field\n |RETURN *\n |\"\"\".stripMargin\n val map: util.Map[String, AnyRef] = Map[String, AnyRef](\"procName\" -> options.query.value).asJava\n val fields = session.run(query, map, sessionTransactionConfig).list.asScala\n .map(r => r.get(\"field\").asList((t: Value) => t.asString()).asScala)\n .map(r =>\n (\n r.head.trim,\n r(1).replaceAll(\"\\\\?\", \"\") match {\n case \"STRING\" => (\"String\", null)\n case \"INTEGER\" => (\"Long\", null)\n case \"FLOAT\" | \"NUMBER\" => (\"Double\", null)\n case \"DATETIME\" => (\"DateTime\", null)\n case \"BOOLEAN\" => (\"Boolean\", null)\n case \"LOCALTIME\" => (\"LocalTime\", null)\n case \"LIST OF INTEGER\" | \"LIST\" | \"LIST\" => (\"LongArray\", null)\n case \"LIST OF FLOAT\" | \"LIST\" | \"LIST\" => (\"DoubleArray\", null)\n case \"LIST OF STRING\" | \"LIST\" | \"LIST\" => (\"StringArray\", null)\n case \"MAP\" =>\n logWarning(\n s\"\"\"\n |For procedure ${options.query.value}\n |Neo4j return type MAP? of field ${r.head.trim} not fully supported.\n |We'll coerce it to a Map\n |\"\"\".stripMargin\n )\n (\"Map\", Map(\"key\" -> \"\").asJava) // dummy value\n case \"LIST OF MAP\" | \"LIST\" | \"LIST\" =>\n logWarning(\n s\"\"\"\n |For procedure ${options.query.value}\n |Neo4j return type LIST? OF MAP? of field ${r.head.trim} not fully supported.\n |We'll coerce it to a [Map]\n |\"\"\".stripMargin\n )\n (\"MapArray\", Seq(Map(\"key\" -> \"\").asJava).asJava) // dummy value\n case \"PATH\" => (\"Path\", null)\n case _ => throw new IllegalArgumentException(s\"Neo4j type ${r(1)} not supported\")\n }\n )\n )\n .map(r => StructField(r._1, cypherToSparkTypeConverter.convert(r._2._1, r._2._2)))\n .toSeq\n StructType(fields)\n }\n\n def inputForGDSProc(procName: String): Seq[(String, Boolean)] = {\n val query =\n \"\"\"\n |WITH $procName AS procName\n |CALL gds.list() YIELD name, signature, type\n |WHERE name = procName AND type = 'procedure'\n |WITH replace(signature, procName + '(', '') AS signature\n |WITH split(signature, ') :: (')[0] AS fields\n |WITH substring(fields, 0, size(fields) - 1) AS fields\n |WITH split(fields, ',') AS fields\n |WITH [field IN fields | split(field, ' :: ')] AS fields\n |UNWIND fields AS field\n |WITH trim(split(field[0], ' = ')[0]) AS fieldName, field[0] contains ' = ' AS optional\n |RETURN *\n |\"\"\".stripMargin\n val map: util.Map[String, AnyRef] = Map[String, AnyRef](\"procName\" -> procName).asJava\n session.run(query, map, sessionTransactionConfig)\n .list\n .asScala\n .map(r => (r.get(\"fieldName\").asString(), r.get(\"optional\").asBoolean()))\n .toSeq\n }\n\n private def getReturnedColumns(query: String): Array[String] =\n session.run(\"EXPLAIN \" + query, sessionTransactionConfig)\n .keys().asScala.toArray\n\n def struct(): StructType = {\n val struct = options.query.queryType match {\n case QueryType.LABELS => structForNode()\n case QueryType.RELATIONSHIP => structForRelationship()\n case QueryType.QUERY => structForQuery()\n case QueryType.GDS => structForGDS()\n }\n\n struct\n }\n\n def countForNodeWithQuery(filters: Array[Filter]): Long = {\n val query = if (filters.isEmpty) {\n options.nodeMetadata.labels\n .map(_.quote())\n .map(label =>\n s\"\"\"\n |MATCH (:$label)\n |RETURN count(*) AS count\"\"\".stripMargin\n )\n .mkString(\" UNION ALL \")\n } else {\n queryReadStrategy.createStatementForNodeCount(options)\n }\n log.info(s\"Executing the following counting query on Neo4j: $query\")\n session.readTransaction(\n tx =>\n tx.run(query, Values.value(Neo4jUtil.paramsFromFilters(filters).asJava))\n .list()\n .asScala\n .map(_.get(\"count\"))\n .map(count => if (count.isNull) 0L else count.asLong())\n .min,\n sessionTransactionConfig\n )\n }\n\n def countForRelationshipWithQuery(filters: Array[Filter]): Long = {\n val query = if (filters.isEmpty) {\n val sourceQueries = options.relationshipMetadata.source.labels\n .map(_.quote())\n .map(label =>\n s\"\"\"MATCH (:$label)-[${Neo4jUtil.RELATIONSHIP_ALIAS}:${options.relationshipMetadata.relationshipType.quote()}]->()\n |RETURN count(${Neo4jUtil.RELATIONSHIP_ALIAS}) AS count\n |\"\"\".stripMargin\n )\n val targetQueries = options.relationshipMetadata.target.labels\n .map(_.quote())\n .map(label =>\n s\"\"\"MATCH ()-[${Neo4jUtil.RELATIONSHIP_ALIAS}:${options.relationshipMetadata.relationshipType.quote()}]->(:$label)\n |RETURN count(${Neo4jUtil.RELATIONSHIP_ALIAS}) AS count\n |\"\"\".stripMargin\n )\n (sourceQueries ++ targetQueries)\n .mkString(\" UNION ALL \")\n } else {\n queryReadStrategy.createStatementForRelationshipCount(options)\n }\n log.info(s\"Executing the following counting query on Neo4j: $query\")\n session.run(query, sessionTransactionConfig)\n .list()\n .asScala\n .map(_.get(\"count\"))\n .map(count => if (count.isNull) 0L else count.asLong())\n .min\n }\n\n def countForNode(filters: Array[Filter]): Long =\n try {\n /*\n * we try to leverage the count store in order to have the faster response possible\n * https://neo4j.com/developer/kb/fast-counts-using-the-count-store/\n * so in this scenario we have some limitations given the fact that we get the min\n * for the sequence of counts returned\n */\n if (filters.isEmpty) {\n val query = \"CALL apoc.meta.stats() yield labels RETURN labels\"\n val map = session.run(query, sessionTransactionConfig).single()\n .asMap()\n .asScala\n .get(\"labels\")\n .getOrElse(Collections.emptyMap())\n .asInstanceOf[util.Map[String, Long]].asScala\n map.filterKeys(k => options.nodeMetadata.labels.contains(k))\n .values.min\n } else {\n countForNodeWithQuery(filters)\n }\n } catch {\n case e: ClientException => {\n logResolutionChange(\"Switching to query count resolution\", e)\n countForNodeWithQuery(filters)\n }\n case e: Throwable => logExceptionForCount(e)\n }\n\n def countForRelationship(filters: Array[Filter]): Long =\n try {\n if (filters.isEmpty) {\n val query = \"CALL apoc.meta.stats() yield relTypes RETURN relTypes\"\n val map = session.run(query, sessionTransactionConfig).single()\n .asMap()\n .asScala\n .get(\"relTypes\")\n .getOrElse(Collections.emptyMap())\n .asInstanceOf[util.Map[String, Long]]\n .asScala\n val minFromSource = options.relationshipMetadata.source.labels\n .map(_.quote())\n .map(label =>\n map.get(s\"(:$label)-[:${options.relationshipMetadata.relationshipType}]->()\").getOrElse(Long.MaxValue)\n )\n .min\n val minFromTarget = options.relationshipMetadata.target.labels\n .map(_.quote())\n .map(label =>\n map.get(s\"()-[:${options.relationshipMetadata.relationshipType}]->(:$label)\").getOrElse(Long.MaxValue)\n )\n .min\n Math.min(minFromSource, minFromTarget)\n } else {\n countForRelationshipWithQuery(filters)\n }\n } catch {\n case e: ClientException => {\n logResolutionChange(\"Switching to query count resolution\", e)\n countForRelationshipWithQuery(filters)\n }\n case e: Throwable => logExceptionForCount(e)\n }\n\n private def logExceptionForCount(e: Throwable): Long = {\n log.error(\"Cannot compute the count because the following exception:\", e)\n -1\n }\n\n def skipLimitFromPartition(topN: Option[TopN]): Seq[PartitionPagination] =\n if (options.partitions == 1) {\n val skipLimit = topN.map(top => PartitionPagination(0, 0, top)).getOrElse(PartitionPagination.EMPTY)\n Seq(skipLimit)\n } else {\n val count: Long = this.count()\n if (count <= 0) {\n Seq(PartitionPagination.EMPTY)\n } else {\n val partitionSize = Math.ceil(count.toDouble / options.partitions).toLong\n (0 until options.partitions)\n .map(index => PartitionPagination(index, index * partitionSize, TopN(partitionSize)))\n }\n }\n\n def count(filters: Array[Filter] = this.filters): Long = options.query.queryType match {\n case QueryType.LABELS => countForNode(filters)\n case QueryType.RELATIONSHIP => countForRelationship(filters)\n case QueryType.QUERY => countForQuery()\n }\n\n private def countForQuery(): Long = {\n val queryCount: String = options.queryMetadata.queryCount\n if (Neo4jUtil.isLong(queryCount)) {\n queryCount.trim.toLong\n } else {\n val query = if (queryCount.nonEmpty) {\n options.queryMetadata.queryCount\n } else {\n s\"\"\"CALL { ${options.query.value} }\n |RETURN count(*) AS count\n |\"\"\".stripMargin\n }\n session.run(query, sessionTransactionConfig).single().get(\"count\").asLong()\n }\n }\n\n def isGdsProcedure(procName: String): Boolean = {\n val params: util.Map[String, AnyRef] = Map[String, AnyRef](\"procName\" -> procName).asJava\n session.run(\n \"\"\"\n |CALL gds.list() YIELD name, type\n |WHERE name = $procName AND type = 'procedure'\n |RETURN count(*) = 1\n |\"\"\".stripMargin,\n params,\n sessionTransactionConfig\n )\n .single()\n .get(0)\n .asBoolean()\n }\n\n def validateQuery(query: String, expectedQueryTypes: org.neo4j.driver.summary.QueryType*): String =\n try {\n val queryType = session.run(s\"EXPLAIN $query\", sessionTransactionConfig).consume().queryType()\n if (expectedQueryTypes.isEmpty || expectedQueryTypes.contains(queryType)) {\n \"\"\n } else {\n s\"Invalid query `${cleanQuery(query)}` because the accepted types are [${expectedQueryTypes.mkString(\", \")}], but the actual type is $queryType\"\n }\n } catch {\n case e: Throwable => s\"Query not compiled for the following exception: ${ExceptionUtils.getMessage(e)}\"\n }\n\n private def cleanQuery(query: String) = {\n query\n .replace(\n s\"WITH {} AS ${Neo4jQueryStrategy.VARIABLE_EVENT}, [] as ${Neo4jQueryStrategy.VARIABLE_SCRIPT_RESULT}\",\n \"\"\n )\n .replace(s\"WITH [] as ${Neo4jQueryStrategy.VARIABLE_SCRIPT_RESULT}\", \"\")\n .replace(s\"WITH {} AS ${Neo4jQueryStrategy.VARIABLE_EVENT}\", \"\")\n .trim\n }\n\n def validateQueryCount(query: String): String =\n try {\n val resultSummary = session.run(s\"EXPLAIN $query\", sessionTransactionConfig).consume()\n val queryType = resultSummary.queryType()\n val plan = resultSummary.plan()\n val expectedQueryTypes =\n Set(org.neo4j.driver.summary.QueryType.READ_ONLY, org.neo4j.driver.summary.QueryType.SCHEMA_WRITE)\n val isReadOnly = expectedQueryTypes.contains(queryType)\n val hasCountIdentifier = plan.identifiers().asScala.toSet == Set(\"count\")\n if (isReadOnly && hasCountIdentifier) {\n \"\"\n } else {\n s\"Invalid query `${cleanQuery(query)}` because the expected type should be [${expectedQueryTypes.mkString(\", \")}], but the actual type is $queryType\"\n }\n } catch {\n case e: Throwable => s\"Query count not compiled for the following exception: ${ExceptionUtils.getMessage(e)}\"\n }\n\n def isValidQuery(query: String, expectedQueryTypes: org.neo4j.driver.summary.QueryType*): Boolean =\n try {\n val queryType = session.run(s\"EXPLAIN $query\", sessionTransactionConfig).consume().queryType()\n expectedQueryTypes.isEmpty || expectedQueryTypes.contains(queryType)\n } catch {\n case e: Throwable => {\n if (log.isDebugEnabled) {\n log.debug(\"Query not compiled because of the following exception:\", e)\n }\n false\n }\n }\n\n @deprecated(\"use createEntityConstraint instead\")\n private def createIndexOrConstraint(action: OptimizationType.Value, label: String, props: Seq[String]): Unit =\n action match {\n case OptimizationType.NONE => log.info(\"No optimization type provided\")\n case _ => {\n try {\n val quotedLabel = label.quote()\n val quotedProps = props\n .map(prop => s\"${Neo4jUtil.NODE_ALIAS}.${prop.quote()}\")\n .mkString(\", \")\n val isNeo4j4 = neo4j.getVersion.getMajor == 4\n val uniqueFieldName = if (!isNeo4j4) \"owningConstraint\" else \"uniqueness\"\n val dashSeparatedProps = props.mkString(\"-\")\n val (querySuffix, uniqueCondition) = action match {\n case OptimizationType.INDEX => (\n s\"FOR (${Neo4jUtil.NODE_ALIAS}:$quotedLabel) ON ($quotedProps)\",\n if (!isNeo4j4) s\"$uniqueFieldName IS NULL\" else s\"$uniqueFieldName = 'NONUNIQUE'\"\n )\n case OptimizationType.NODE_CONSTRAINTS => {\n val assertType = if (props.size > 1) \"NODE KEY\" else \"UNIQUE\"\n (\n s\"FOR (${Neo4jUtil.NODE_ALIAS}:$quotedLabel) REQUIRE ($quotedProps) IS $assertType\",\n if (!isNeo4j4) s\"$uniqueFieldName IS NOT NULL\" else s\"$uniqueFieldName = 'UNIQUE'\"\n )\n }\n }\n val actionName = s\"spark_${action.toString}_${label}_$dashSeparatedProps\".quote()\n val queryPrefix = action match {\n case OptimizationType.INDEX => s\"CREATE INDEX $actionName\"\n case OptimizationType.NODE_CONSTRAINTS => s\"CREATE CONSTRAINT $actionName\"\n }\n val queryCheck =\n s\"\"\"SHOW INDEXES YIELD labelsOrTypes, properties, $uniqueFieldName\n |WHERE labelsOrTypes = ${'$'}labels\n |AND properties = ${'$'}properties\n |AND $uniqueCondition\n |RETURN count(*) > 0 AS isPresent\"\"\".stripMargin\n val params: util.Map[String, AnyRef] = Map(\n \"labels\" -> Seq(label).asJava,\n \"properties\" -> props.asJava\n ).asJava.asInstanceOf[util.Map[String, AnyRef]]\n val isPresent = session.run(queryCheck, params, sessionTransactionConfig)\n .single()\n .get(\"isPresent\")\n .asBoolean()\n\n val status = if (isPresent) {\n \"KEPT\"\n } else {\n val query = s\"$queryPrefix $querySuffix\"\n log.info(s\"Performing the following schema query: $query\")\n session.run(query, sessionTransactionConfig)\n \"CREATED\"\n }\n log.info(s\"Status for $action named with label $quotedLabel and props $quotedProps is: $status\")\n } catch {\n case e: Throwable => log.info(\"Cannot perform the optimization query because of the following exception:\", e)\n }\n }\n }\n\n private def createEntityConstraint(\n entityType: String,\n entityIdentifier: String,\n constraintsOptimizationType: ConstraintsOptimizationType.Value,\n keys: Map[String, String]\n ): Unit = {\n val constraintType = if (constraintsOptimizationType == ConstraintsOptimizationType.UNIQUE) {\n \"UNIQUE\"\n } else {\n s\"$entityType KEY\"\n }\n val dashSeparatedProps = keys.values.mkString(\"-\")\n val constraintName =\n s\"spark_${entityType}_${constraintType.replace(s\"$entityType \", \"\")}-CONSTRAINT_${entityIdentifier}_$dashSeparatedProps\".quote()\n val props = keys.values.map(_.quote()).map(\"e.\" + _).mkString(\", \")\n val asciiRepresentation: String = createCypherPattern(entityType, entityIdentifier)\n session.writeTransaction(\n tx => {\n tx.run(\n s\"CREATE CONSTRAINT $constraintName IF NOT EXISTS FOR $asciiRepresentation REQUIRE ($props) IS $constraintType\"\n )\n },\n sessionTransactionConfig\n )\n }\n\n private def createCypherPattern(entityType: String, entityIdentifier: String) = {\n val asciiRepresentation = entityType match {\n case \"NODE\" => s\"(e:${entityIdentifier.quote()})\"\n case \"RELATIONSHIP\" => s\"()-[e:${entityIdentifier.quote()}]->()\"\n case _ => throw new IllegalArgumentException(s\"$entityType not supported\")\n }\n asciiRepresentation\n }\n\n private def createEntityTypeConstraint(\n entityType: String,\n entityIdentifier: String,\n properties: Map[String, String],\n struct: StructType,\n constraints: Set[SchemaConstraintsOptimizationType.Value]\n ): Unit = {\n val asciiRepresentation: String = createCypherPattern(entityType, entityIdentifier)\n session.writeTransaction(\n tx => {\n properties\n .filter(t => struct.exists(f => f.name == t._1))\n .map(t => {\n val field = struct.find(f => f.name == t._1).get\n (t._2, sparkToCypherTypeConverter.convert(field.dataType), field.nullable)\n })\n .foreach(t => {\n val prop = t._1.quote()\n val cypherType = t._2\n val isNullable = t._3\n if (constraints.contains(SchemaConstraintsOptimizationType.TYPE)) {\n val typeConstraintName = s\"spark_$entityType-TYPE-CONSTRAINT-$entityIdentifier-$prop\".quote()\n tx.run(\n s\"CREATE CONSTRAINT $typeConstraintName IF NOT EXISTS FOR $asciiRepresentation REQUIRE e.$prop IS :: $cypherType\"\n ).consume()\n }\n if (constraints.contains(SchemaConstraintsOptimizationType.EXISTS)) {\n if (!isNullable) {\n val notNullConstraintName = s\"spark_$entityType-NOT_NULL-CONSTRAINT-$entityIdentifier-$prop\".quote()\n tx.run(\n s\"CREATE CONSTRAINT $notNullConstraintName IF NOT EXISTS FOR $asciiRepresentation REQUIRE e.$prop IS NOT NULL\"\n ).consume()\n }\n }\n })\n },\n sessionTransactionConfig\n )\n }\n\n private def createOptimizationsForNode(struct: StructType): Unit = {\n val schemaMetadata = options.schemaMetadata.optimization\n if (\n schemaMetadata.nodeConstraint != ConstraintsOptimizationType.NONE\n || schemaMetadata.schemaConstraints != Set(SchemaConstraintsOptimizationType.NONE)\n ) {\n if (schemaMetadata.nodeConstraint != ConstraintsOptimizationType.NONE) {\n createEntityConstraint(\n \"NODE\",\n options.nodeMetadata.labels.head,\n schemaMetadata.nodeConstraint,\n options.nodeMetadata.nodeKeys\n )\n }\n if (schemaMetadata.schemaConstraints.nonEmpty) {\n val propsFromStruct: Map[String, String] = struct\n .map(f => (f.name, f.name))\n .toMap\n val propsFromMeta: Map[String, String] = options.nodeMetadata.nodeKeys ++ options.nodeMetadata.properties\n createEntityTypeConstraint(\n \"NODE\",\n options.nodeMetadata.labels.head,\n propsFromStruct ++ propsFromMeta,\n struct,\n schemaMetadata.schemaConstraints\n )\n }\n } else { // TODO old behaviour, remove it in the future\n options.schemaMetadata.optimizationType match {\n case OptimizationType.INDEX | OptimizationType.NODE_CONSTRAINTS => {\n createIndexOrConstraint(\n options.schemaMetadata.optimizationType,\n options.nodeMetadata.labels.head,\n options.nodeMetadata.nodeKeys.values.toSeq\n )\n }\n case _ => // do nothing\n }\n }\n }\n\n private def createOptimizationsForRelationship(struct: StructType): Unit = {\n val schemaMetadata = options.schemaMetadata.optimization\n if (\n schemaMetadata.nodeConstraint != ConstraintsOptimizationType.NONE\n || schemaMetadata.relConstraint != ConstraintsOptimizationType.NONE\n || schemaMetadata.schemaConstraints != Set(SchemaConstraintsOptimizationType.NONE)\n ) {\n if (schemaMetadata.nodeConstraint != ConstraintsOptimizationType.NONE) {\n createEntityConstraint(\n \"NODE\",\n options.relationshipMetadata.source.labels.head,\n schemaMetadata.nodeConstraint,\n options.relationshipMetadata.source.nodeKeys\n )\n createEntityConstraint(\n \"NODE\",\n options.relationshipMetadata.target.labels.head,\n schemaMetadata.nodeConstraint,\n options.relationshipMetadata.target.nodeKeys\n )\n }\n if (schemaMetadata.relConstraint != ConstraintsOptimizationType.NONE) {\n createEntityConstraint(\n \"RELATIONSHIP\",\n options.relationshipMetadata.relationshipType,\n schemaMetadata.relConstraint,\n options.relationshipMetadata.relationshipKeys\n )\n }\n if (schemaMetadata.schemaConstraints.nonEmpty) {\n val sourceNodeProps: Map[String, String] =\n options.relationshipMetadata.source.nodeKeys ++ options.relationshipMetadata.source.properties\n val targetNodeProps: Map[String, String] =\n options.relationshipMetadata.target.nodeKeys ++ options.relationshipMetadata.target.properties\n val allNodeProps: Map[String, String] = sourceNodeProps ++ targetNodeProps\n val relStruct: StructType = StructType(struct.filterNot(f => allNodeProps.contains(f.name)))\n val propsFromRelStruct: Map[String, String] = relStruct\n .map(f => (f.name, f.name))\n .toMap\n val propsFromMeta: Map[String, String] =\n options.relationshipMetadata.relationshipKeys ++ options.relationshipMetadata.properties.getOrElse(Map.empty)\n createEntityTypeConstraint(\n \"RELATIONSHIP\",\n options.relationshipMetadata.relationshipType,\n propsFromRelStruct ++ propsFromMeta,\n struct,\n schemaMetadata.schemaConstraints\n )\n createEntityTypeConstraint(\n \"NODE\",\n options.relationshipMetadata.source.labels.head,\n sourceNodeProps,\n struct,\n schemaMetadata.schemaConstraints\n )\n createEntityTypeConstraint(\n \"NODE\",\n options.relationshipMetadata.target.labels.head,\n targetNodeProps,\n struct,\n schemaMetadata.schemaConstraints\n )\n }\n } else { // TODO old behaviour, remove it in the future\n options.schemaMetadata.optimizationType match {\n case OptimizationType.INDEX | OptimizationType.NODE_CONSTRAINTS => {\n createIndexOrConstraint(\n options.schemaMetadata.optimizationType,\n options.relationshipMetadata.source.labels.head,\n options.relationshipMetadata.source.nodeKeys.values.toSeq\n )\n createIndexOrConstraint(\n options.schemaMetadata.optimizationType,\n options.relationshipMetadata.target.labels.head,\n options.relationshipMetadata.target.nodeKeys.values.toSeq\n )\n }\n case _ => // do nothing\n }\n }\n }\n\n def createOptimizations(struct: StructType): Unit = {\n Validations.validate(ValidateSchemaOptions(options, struct))\n options.query.queryType match {\n case QueryType.LABELS => createOptimizationsForNode(struct)\n case QueryType.RELATIONSHIP => createOptimizationsForRelationship(struct)\n case _ => // do nothing\n }\n }\n\n def execute(queries: Seq[String]): util.List[util.Map[String, AnyRef]] = {\n val queryMap = queries\n .map(query => {\n (session.run(s\"EXPLAIN $query\", sessionTransactionConfig).consume().queryType(), query)\n })\n .groupBy(_._1)\n .mapValues(_.map(_._2))\n val schemaQueries = queryMap.getOrElse(org.neo4j.driver.summary.QueryType.SCHEMA_WRITE, Seq.empty[String])\n schemaQueries.foreach(session.run(_, sessionTransactionConfig))\n val others = queryMap\n .filterKeys(key => key != org.neo4j.driver.summary.QueryType.SCHEMA_WRITE)\n .values\n .flatten\n .toSeq\n if (others.isEmpty) {\n Collections.emptyList()\n } else {\n session\n .writeTransaction(\n new TransactionWork[util.List[java.util.Map[String, AnyRef]]] {\n override def execute(transaction: Transaction): util.List[util.Map[String, AnyRef]] = {\n others.size match {\n case 1 => transaction.run(others.head).list()\n .asScala\n .map(_.asMap())\n .asJava\n case _ => {\n others\n .slice(0, queries.size - 1)\n .foreach(transaction.run)\n val result = transaction.run(others.last).list()\n .asScala\n .map(_.asMap())\n .asJava\n result\n }\n }\n }\n },\n sessionTransactionConfig\n )\n }\n }\n\n def lastOffset(): Option[Long] = options.query.queryType match {\n case QueryType.LABELS => lastOffsetForNode()\n case QueryType.RELATIONSHIP => lastOffsetForRelationship()\n case QueryType.QUERY => lastOffsetForQuery()\n }\n\n private def lastOffsetForNode(): Option[Long] = {\n val label = options.nodeMetadata.labels.head\n session.run(\n s\"\"\"MATCH (n:$label)\n |RETURN max(n.${options.streamingOptions.propertyName}) AS ${options.streamingOptions.propertyName}\"\"\".stripMargin,\n sessionTransactionConfig\n )\n .single()\n .get(options.streamingOptions.propertyName)\n .asOptionalLong()\n }\n\n private def lastOffsetForRelationship(): Option[Long] = {\n val sourceLabel = options.relationshipMetadata.source.labels.head.quote()\n val targetLabel = options.relationshipMetadata.target.labels.head.quote()\n val relType = options.relationshipMetadata.relationshipType.quote()\n\n session.run(\n s\"\"\"MATCH (s:$sourceLabel)-[r:$relType]->(t:$targetLabel)\n |RETURN max(r.${options.streamingOptions.propertyName}) AS ${options.streamingOptions.propertyName}\"\"\".stripMargin,\n sessionTransactionConfig\n )\n .single()\n .get(options.streamingOptions.propertyName)\n .asOptionalLong()\n }\n\n private def lastOffsetForQuery(): Option[Long] = {\n session.run(options.streamingOptions.queryOffset, sessionTransactionConfig)\n .single()\n .get(0)\n .asOptionalLong()\n }\n\n private def logResolutionChange(message: String, e: ClientException): Unit = {\n log.warn(message)\n if (!e.code().equals(\"Neo.ClientError.Procedure.ProcedureNotFound\")) {\n log.warn(s\"For the following exception\", e)\n }\n }\n\n override def close(): Unit = {\n Neo4jUtil.closeSafely(session, log)\n }\n\n}\n\nobject SchemaService {\n val POINT_TYPE_2D = \"point-2d\"\n val POINT_TYPE_3D = \"point-3d\"\n\n val TIME_TYPE_OFFSET = \"offset-time\"\n val TIME_TYPE_LOCAL = \"local-time\"\n\n val DURATION_TYPE = \"duration\"\n\n def normalizedClassName(value: AnyRef, options: Neo4jOptions): String = value match {\n case binary: Array[Byte] => if (options.legacyTypeConversionEnabled) value.getClass.getSimpleName else \"ByteArray\"\n case list: java.util.List[_] => \"Array\"\n case map: java.util.Map[String, _] => \"Map\"\n case null => \"String\"\n case _ => value.getClass.getSimpleName\n }\n\n // from nodes and relationships we cannot have maps as properties and elements in lists are the same type\n // special treatment for ByteArray required (pattern matching on Array != List)\n def normalizedClassNameFromGraphEntity(value: AnyRef, options: Neo4jOptions): String = value match {\n case binary: Array[Byte] => if (options.legacyTypeConversionEnabled) value.getClass.getSimpleName else \"ByteArray\"\n case list: java.util.List[_] => s\"${list.get(0).getClass.getSimpleName}Array\"\n case null => \"String\"\n case _ => value.getClass.getSimpleName\n }\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/streaming/BaseStreamingPartitionReader.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.streaming\n\nimport org.apache.spark.sql.connector.expressions.aggregate.AggregateFunc\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.sources.GreaterThan\nimport org.apache.spark.sql.sources.LessThanOrEqual\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.cypherdsl.core.Cypher\nimport org.neo4j.spark.reader.BasePartitionReader\nimport org.neo4j.spark.service.Neo4jQueryStrategy\nimport org.neo4j.spark.service.PartitionPagination\nimport org.neo4j.spark.streaming.BaseStreamingPartitionReader.offsetUsagePatterns\nimport org.neo4j.spark.util.Neo4jImplicits._\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.Neo4jUtil\nimport org.neo4j.spark.util.QueryType._\nimport org.neo4j.spark.util.StreamingFrom\n\nimport java.util\nimport java.util.function.Predicate\nimport java.util.regex.Pattern\n\nimport scala.collection.JavaConverters.mapAsJavaMapConverter\n\nclass BaseStreamingPartitionReader(\n private val neo4j: Neo4j,\n private val options: Neo4jOptions,\n private val filters: Array[Filter],\n private val schema: StructType,\n private val jobId: String,\n private val partitionSkipLimit: PartitionPagination,\n private val scriptResult: java.util.List[java.util.Map[String, AnyRef]],\n private val requiredColumns: StructType,\n private val aggregateColumns: Array[AggregateFunc]\n) extends BasePartitionReader(\n neo4j,\n options,\n filters,\n schema,\n jobId,\n partitionSkipLimit,\n scriptResult,\n requiredColumns,\n aggregateColumns\n ) {\n\n private val streamingPropertyName = Neo4jUtil.getStreamingPropertyName(options)\n\n private val streamStart =\n filters.find(f => f.getAttribute.contains(streamingPropertyName) && f.isInstanceOf[GreaterThan])\n\n private val streamEnd =\n filters.find(f => f.getAttribute.contains(streamingPropertyName) && f.isInstanceOf[LessThanOrEqual])\n\n logInfo(s\"Creating Streaming Partition reader $name\")\n\n private lazy val values = {\n val map = new util.HashMap[String, Any](super.queryParameters)\n val start: Long = streamStart\n .flatMap(f => f.getValue)\n .getOrElse(StreamingFrom.ALL.value())\n .asInstanceOf[Long]\n val end: Long = streamEnd\n .flatMap(f => f.getValue)\n .get\n .asInstanceOf[Long]\n map.put(Neo4jQueryStrategy.VARIABLE_STREAM, Map(\"offset\" -> start, \"from\" -> start, \"to\" -> end).asJava)\n map\n }\n\n override def close(): Unit = {\n logInfo(s\"Closing Partition reader $name ${if (hasError()) \"with error \" else \"\"}\")\n super.close()\n }\n\n override protected def query(): String = {\n options.query.queryType match {\n case QUERY =>\n val originalQuery = super.query()\n\n if (offsetUsagePatterns.exists(_.test(originalQuery))) {\n logWarning(\n \"Usage of '$stream.offset' is deprecated in favor of '$stream.from' and '$stream.to' parameters which \"\n + \"describes the range of changes the micro batch refers to. Please update your queries accordingly.\"\n )\n }\n\n val property = Cypher.name(streamingPropertyName)\n val stream = Cypher.parameter(\"stream\")\n\n // rewrite query for adding $stream.from and $stream.to filters\n Cypher.callRawCypher(originalQuery)\n .`with`(Cypher.asterisk())\n .where(\n property.gt(stream.property(\"from\")).and(property.lte(stream.property(\"to\")))\n )\n .returning(Cypher.asterisk())\n .build()\n .getCypher\n // we don't need to rewrite the queries for LABELS and RELATIONSHIPS because spark filters already cover our\n // criteria which are added to the query text in Neo4jQueryService\n case LABELS => super.query()\n case RELATIONSHIP => super.query()\n case GDS =>\n throw new UnsupportedOperationException(\"GDS strategy is not supported in structured streaming use cases.\")\n }\n }\n\n override protected def queryParameters: util.Map[String, Any] = values\n\n}\n\nobject BaseStreamingPartitionReader {\n\n private val offsetUsagePatterns: Seq[Predicate[String]] = Seq(\n Pattern.compile(\"\\\\$stream\\\\.offset\").asPredicate(),\n Pattern.compile(\"\\\\$`stream`\\\\.offset\").asPredicate(),\n Pattern.compile(\"\\\\$stream\\\\.`offset`\").asPredicate(),\n Pattern.compile(\"\\\\$`stream`\\\\.`offset`\").asPredicate()\n )\n\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/util/DriverCache.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport org.neo4j.driver.Driver\nimport org.neo4j.spark.util.DriverCache.cache\n\nimport java.util.concurrent.ConcurrentHashMap\nimport java.util.concurrent.atomic.AtomicInteger\n\nobject DriverCache {\n\n private val cache: ConcurrentHashMap[Neo4jDriverOptions, (Driver, AtomicInteger)] =\n new ConcurrentHashMap[Neo4jDriverOptions, (Driver, AtomicInteger)]\n}\n\nclass DriverCache(private val options: Neo4jDriverOptions) extends Serializable\n with AutoCloseable {\n\n def getOrCreate(): Driver = {\n val (driver, counter) =\n cache.computeIfAbsent(options, (t: Neo4jDriverOptions) => (t.createDriver(), new AtomicInteger(0)))\n counter.incrementAndGet()\n driver\n }\n\n def close(): Unit = {\n val (driver, counter) = cache.get(options)\n if (counter.decrementAndGet() == 0) {\n cache.remove(options)\n Neo4jUtil.closeSafely(driver)\n }\n }\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/util/Neo4jImplicits.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport com.fasterxml.jackson.core.JsonParseException\nimport com.fasterxml.jackson.core.JsonParser\nimport com.fasterxml.jackson.databind.ObjectMapper\nimport org.apache.spark.sql.connector.expressions.Expression\nimport org.apache.spark.sql.connector.expressions.Literal\nimport org.apache.spark.sql.connector.expressions.aggregate.Aggregation\nimport org.apache.spark.sql.connector.expressions.filter\nimport org.apache.spark.sql.connector.expressions.filter.Predicate\nimport org.apache.spark.sql.sources.AlwaysFalse\nimport org.apache.spark.sql.sources.AlwaysTrue\nimport org.apache.spark.sql.sources.And\nimport org.apache.spark.sql.sources.EqualNullSafe\nimport org.apache.spark.sql.sources.EqualTo\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.sources.GreaterThan\nimport org.apache.spark.sql.sources.GreaterThanOrEqual\nimport org.apache.spark.sql.sources.In\nimport org.apache.spark.sql.sources.IsNotNull\nimport org.apache.spark.sql.sources.IsNull\nimport org.apache.spark.sql.sources.LessThan\nimport org.apache.spark.sql.sources.LessThanOrEqual\nimport org.apache.spark.sql.sources.Not\nimport org.apache.spark.sql.sources.Or\nimport org.apache.spark.sql.sources.StringContains\nimport org.apache.spark.sql.sources.StringEndsWith\nimport org.apache.spark.sql.sources.StringStartsWith\nimport org.apache.spark.sql.types.DataTypes\nimport org.apache.spark.sql.types.MapType\nimport org.apache.spark.sql.types.StructField\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.driver.Value\nimport org.neo4j.driver.types.Entity\nimport org.neo4j.driver.types.Node\nimport org.neo4j.driver.types.Relationship\nimport org.neo4j.spark.converter.CypherToSparkTypeConverter\nimport org.neo4j.spark.converter.SparkToNeo4jDataConverter\nimport org.neo4j.spark.service.SchemaService\n\nimport scala.collection.JavaConverters._\n\nimport javax.lang.model.SourceVersion\n\nobject Neo4jImplicits {\n\n implicit class CypherImplicits(str: String) {\n private def isValidCypherIdentifier() = SourceVersion.isIdentifier(str) && !str.trim.startsWith(\"$\")\n\n def quote(): String = if (!isValidCypherIdentifier() && !str.isQuoted()) s\"`$str`\" else str\n\n def unquote(): String = str.replaceAll(\"`\", \"\");\n\n def isQuoted(): Boolean = str.startsWith(\"`\");\n\n def removeAlias(): String = {\n val splatString = str.unquote().split('.')\n\n if (splatString.size > 1) {\n splatString.tail.mkString(\".\")\n } else {\n str\n }\n }\n\n /**\n * df: we need this to handle scenarios like `WHERE age > 19 and age < 22`,\n * so we can't basically add a parameter named \\$age.\n * So we base64 encode the value to ensure a unique parameter name\n */\n def toParameterName(value: Any): String = {\n val attributeValue = if (value == null) {\n \"NULL\"\n } else {\n value.toString\n }\n\n val base64ed = java.util.Base64.getEncoder.encodeToString(attributeValue.getBytes())\n\n s\"${base64ed}_${str.unquote()}\".quote()\n }\n }\n\n implicit class EntityImplicits(entity: Entity) {\n\n def toStruct(options: Neo4jOptions): StructType = {\n val fields = entity.asMap().asScala\n .groupBy(_._1)\n .map(t => {\n val value = t._2.head._2\n val cypherType = SchemaService.normalizedClassNameFromGraphEntity(value, options)\n StructField(t._1, CypherToSparkTypeConverter(options).convert(cypherType))\n })\n val entityFields = entity match {\n case _: Node => {\n Seq(\n StructField(Neo4jUtil.INTERNAL_ID_FIELD, DataTypes.LongType, nullable = false),\n StructField(\n Neo4jUtil.INTERNAL_LABELS_FIELD,\n DataTypes.createArrayType(DataTypes.StringType),\n nullable = true\n )\n )\n }\n case _: Relationship => {\n Seq(\n StructField(Neo4jUtil.INTERNAL_REL_ID_FIELD, DataTypes.LongType, nullable = false),\n StructField(Neo4jUtil.INTERNAL_REL_TYPE_FIELD, DataTypes.StringType, nullable = false),\n StructField(Neo4jUtil.INTERNAL_REL_SOURCE_ID_FIELD, DataTypes.LongType, nullable = false),\n StructField(Neo4jUtil.INTERNAL_REL_TARGET_ID_FIELD, DataTypes.LongType, nullable = false)\n )\n }\n }\n\n StructType(entityFields ++ fields)\n }\n\n def toMap: java.util.Map[String, Any] = {\n val entityMap = entity.asMap().asScala\n val entityFields = entity match {\n case node: Node => {\n Map(Neo4jUtil.INTERNAL_ID_FIELD -> node.id(), Neo4jUtil.INTERNAL_LABELS_FIELD -> node.labels())\n }\n case relationship: Relationship => {\n Map(\n Neo4jUtil.INTERNAL_REL_ID_FIELD -> relationship.id(),\n Neo4jUtil.INTERNAL_REL_TYPE_FIELD -> relationship.`type`(),\n Neo4jUtil.INTERNAL_REL_SOURCE_ID_FIELD -> relationship.startNodeId(),\n Neo4jUtil.INTERNAL_REL_TARGET_ID_FIELD -> relationship.endNodeId()\n )\n }\n }\n (entityFields ++ entityMap).asJava\n }\n }\n\n implicit class PredicateImplicit(predicate: Predicate) {\n\n def toFilter(options: Neo4jOptions): Option[Filter] = {\n predicate.name() match {\n case \"IS_NULL\" => Some(IsNull(predicate.rawAttributeName()))\n case \"IS_NOT_NULL\" => Some(IsNotNull(predicate.rawAttributeName()))\n case \"STARTS_WITH\" =>\n predicate.rawLiteralValue(options).map(lit => StringStartsWith(predicate.rawAttributeName(), lit.asString()))\n case \"ENDS_WITH\" =>\n predicate.rawLiteralValue(options).map(lit => StringEndsWith(predicate.rawAttributeName(), lit.asString()))\n case \"CONTAINS\" =>\n predicate.rawLiteralValue(options).map(lit => StringContains(predicate.rawAttributeName(), lit.asString()))\n case \"IN\" => Some(In(predicate.rawAttributeName(), predicate.rawLiteralValues(options)))\n case \"=\" => predicate.rawLiteralValue(options).map(lit => EqualTo(predicate.rawAttributeName(), lit.asObject()))\n case \"<>\" =>\n predicate.rawLiteralValue(options).map(lit => Not(EqualTo(predicate.rawAttributeName(), lit.asObject())))\n case \"<=>\" =>\n predicate.rawLiteralValue(options).map(lit => EqualNullSafe(predicate.rawAttributeName(), lit.asObject()))\n case \"<\" =>\n predicate.rawLiteralValue(options).map(lit => LessThan(predicate.rawAttributeName(), lit.asObject()))\n case \"<=\" =>\n predicate.rawLiteralValue(options).map(lit => LessThanOrEqual(predicate.rawAttributeName(), lit.asObject()))\n case \">\" =>\n predicate.rawLiteralValue(options).map(lit => GreaterThan(predicate.rawAttributeName(), lit.asObject()))\n case \">=\" =>\n predicate.rawLiteralValue(options).map(lit =>\n GreaterThanOrEqual(predicate.rawAttributeName(), lit.asObject())\n )\n case \"AND\" =>\n val andPredicate = predicate.asInstanceOf[filter.And]\n (andPredicate.left().toFilter(options), andPredicate.right().toFilter(options)) match {\n case (_, None) => None\n case (None, _) => None\n case (Some(left), Some(right)) => Some(And(left, right))\n }\n case \"OR\" =>\n val andPredicate = predicate.asInstanceOf[filter.Or]\n (andPredicate.left().toFilter(options), andPredicate.right().toFilter(options)) match {\n case (_, None) => None\n case (None, _) => None\n case (Some(left), Some(right)) => Some(Or(left, right))\n }\n case \"NOT\" =>\n val notPredicate = predicate.asInstanceOf[filter.Not]\n notPredicate.child().toFilter(options).map(Not)\n case \"ALWAYS_TRUE\" => Some(AlwaysTrue)\n case \"ALWAYS_FALSE\" => Some(AlwaysFalse)\n }\n }\n\n def rawAttributeName(): String = {\n predicate.references().head.fieldNames().mkString(\".\")\n }\n\n def rawLiteralValue(options: Neo4jOptions): Option[Value] = {\n predicate.children()\n .filter(_.isInstanceOf[Literal[_]])\n .map(_.asInstanceOf[Literal[_]])\n .headOption\n .map(literal => SparkToNeo4jDataConverter(options).convert(literal.value(), literal.dataType()))\n }\n\n def rawLiteralValues(options: Neo4jOptions): Array[Any] = {\n predicate.children()\n .filter(_.isInstanceOf[Literal[_]])\n .map(_.asInstanceOf[Literal[_]])\n .map(v => SparkToNeo4jDataConverter(options).convert(v.value(), v.dataType()).asObject())\n }\n }\n\n implicit class FilterImplicit(filter: Filter) {\n\n def flattenFilters: Array[Filter] = {\n filter match {\n case or: Or => Array(or.left.flattenFilters, or.right.flattenFilters).flatten\n case and: And => Array(and.left.flattenFilters, and.right.flattenFilters).flatten\n case f: Filter => Array(f)\n }\n }\n\n def getAttribute: Option[String] = Option(filter match {\n case eqns: EqualNullSafe => eqns.attribute\n case eq: EqualTo => eq.attribute\n case gt: GreaterThan => gt.attribute\n case gte: GreaterThanOrEqual => gte.attribute\n case lt: LessThan => lt.attribute\n case lte: LessThanOrEqual => lte.attribute\n case in: In => in.attribute\n case notNull: IsNotNull => notNull.attribute\n case isNull: IsNull => isNull.attribute\n case startWith: StringStartsWith => startWith.attribute\n case endsWith: StringEndsWith => endsWith.attribute\n case contains: StringContains => contains.attribute\n case not: Not => not.child.getAttribute.orNull\n case _ => null\n })\n\n def getValue: Option[Any] = Option(filter match {\n case eqns: EqualNullSafe => eqns.value\n case eq: EqualTo => eq.value\n case gt: GreaterThan => gt.value\n case gte: GreaterThanOrEqual => gte.value\n case lt: LessThan => lt.value\n case lte: LessThanOrEqual => lte.value\n case in: In => in.values\n case startWith: StringStartsWith => startWith.value\n case endsWith: StringEndsWith => endsWith.value\n case contains: StringContains => contains.value\n case not: Not => not.child.getValue.orNull\n case _ => null\n })\n\n def isAttribute(entityType: String): Boolean = {\n getAttribute.exists(_.contains(s\"$entityType.\"))\n }\n\n def getAttributeWithoutEntityName: Option[String] =\n filter.getAttribute.map(_.unquote().split('.').tail.mkString(\".\"))\n\n /**\n * df: we are not handling AND/OR because they are not actually filters\n * and have a different internal structure. Before calling this function on the filters\n * it's highly suggested FilterImplicit::flattenFilter() which returns a collection\n * of filters, including the one contained in the ANDs/ORs objects.\n */\n def getAttributeAndValue: Seq[Any] = {\n filter match {\n case f: EqualNullSafe => Seq(f.attribute.toParameterName(f.value), f.value)\n case f: EqualTo => Seq(f.attribute.toParameterName(f.value), f.value)\n case f: GreaterThan => Seq(f.attribute.toParameterName(f.value), f.value)\n case f: GreaterThanOrEqual => Seq(f.attribute.toParameterName(f.value), f.value)\n case f: LessThan => Seq(f.attribute.toParameterName(f.value), f.value)\n case f: LessThanOrEqual => Seq(f.attribute.toParameterName(f.value), f.value)\n case f: In => Seq(f.attribute.toParameterName(f.values), f.values)\n case f: StringStartsWith => Seq(f.attribute.toParameterName(f.value), f.value)\n case f: StringEndsWith => Seq(f.attribute.toParameterName(f.value), f.value)\n case f: StringContains => Seq(f.attribute.toParameterName(f.value), f.value)\n case f: Not => f.child.getAttributeAndValue\n case _ => Seq()\n }\n }\n }\n\n implicit class StructTypeImplicit(structType: StructType) {\n\n private def isValidMapOrStructField(field: String, structFieldName: String) = {\n val value: String = \"\"\"(`.*`)|([^\\.]*)\"\"\".r.findFirstIn(field).getOrElse(\"\")\n structFieldName == value.unquote() || structFieldName == value\n }\n\n def getByName(name: String): Option[StructField] = {\n val index = structType.fieldIndex(name)\n if (index > -1) Some(structType(index)) else None\n }\n\n def getFieldIndex(fieldName: String): Long = structType.fields.map(_.name).indexOf(fieldName)\n\n def getMissingFields(fields: Set[String]): Set[String] = fields\n .map(field => {\n val maybeField = structType\n .find(structField => {\n structField.dataType match {\n case _: MapType => isValidMapOrStructField(field, structField.name)\n case _: StructType => isValidMapOrStructField(field, structField.name)\n case _ => structField.name == field.unquote() || structField.name == field\n }\n })\n field -> maybeField.isDefined\n })\n .filterNot(e => e._2)\n .map(e => e._1)\n }\n\n implicit class AggregationImplicit(aggregation: Aggregation) {\n def groupByCols(): Array[Expression] = ReflectionUtils.groupByCols(aggregation)\n }\n\n implicit class MapImplicit[K, V](map: Map[String, V]) {\n\n private def innerFlattenMap(map: Map[String, _], prefix: String): Seq[(String, AnyRef)] = map\n .toSeq\n .flatMap(t => {\n val key: String = if (prefix != \"\") s\"$prefix.${t._1}\" else t._1\n t._2 match {\n case nestedMap: Map[String, _] => innerFlattenMap(nestedMap, key)\n case nestedMap: java.util.Map[String, _] => innerFlattenMap(nestedMap.asScala.toMap, key)\n case _ => Seq((key, t._2.asInstanceOf[AnyRef]))\n }\n })\n .toList\n\n def flattenMap(prefix: String = \"\", groupDuplicateKeys: Boolean = false): Map[String, AnyRef] =\n innerFlattenMap(map, prefix)\n .groupBy(_._1)\n .mapValues(seq => if (groupDuplicateKeys && seq.size > 1) seq.map(_._2).asJava else seq.last._2)\n .toMap\n\n def flattenKeys(prefix: String = \"\"): Seq[String] = map\n .flatMap(t => {\n val key: String = if (prefix != \"\") s\"$prefix.${t._1}\" else t._1\n t._2 match {\n case nestedMap: Map[String, _] => nestedMap.flattenKeys(key)\n case nestedMap: java.util.Map[String, _] => nestedMap.asScala.toMap.flattenKeys(key)\n case _ => Seq(key)\n }\n })\n .toList\n }\n\n implicit class StringMapImplicits(map: Map[String, String]) {\n\n private val propertyMapper = new ObjectMapper()\n propertyMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)\n\n private def nestingMap(data: Map[String, String]): java.util.Map[String, Any] = {\n val map = new java.util.HashMap[String, Any]();\n data.foreach(t => {\n val splitted = t._1.split(\"\\\\.\")\n if (splitted.size == 1) {\n val value =\n try {\n propertyMapper.readValue[Any](t._2, classOf[Any])\n } catch {\n case _: JsonParseException => t._2\n }\n map.put(t._1, value)\n } else {\n if (map.containsKey(splitted.head)) {\n val value = map.get(splitted.head).asInstanceOf[java.util.Map[String, Any]]\n value.putAll(nestingMap(Map(splitted.drop(1).mkString(\".\") -> t._2)))\n map.put(splitted.head, value)\n } else {\n map.put(splitted.head, nestingMap(Map(splitted.drop(1).mkString(\".\") -> t._2)))\n }\n }\n })\n map\n }\n\n def toNestedJavaMap: java.util.Map[String, Any] = nestingMap(map)\n }\n\n implicit class ValueImplicits(value: Value) {\n\n def asOptionalLong(): Option[Long] = {\n if (value.isNull) {\n Option.empty\n } else {\n Option(value.asLong())\n }\n }\n }\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/util/Neo4jOptions.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport org.apache.spark.internal.Logging\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.SparkSession\nimport org.jetbrains.annotations.TestOnly\nimport org.neo4j.connectors.authn.AuthenticationToken\nimport org.neo4j.connectors.authn.AuthenticationTokenSupplierFactory\nimport org.neo4j.connectors.common.driver.reauth.ReAuthDriverFactory\nimport org.neo4j.driver.Config.TrustStrategy\nimport org.neo4j.driver._\nimport org.neo4j.driver.exceptions.Neo4jException\nimport org.neo4j.driver.net.ServerAddress\nimport org.neo4j.spark.util.Neo4jImplicits.StringMapImplicits\n\nimport java.io.File\nimport java.net.URI\nimport java.time.Duration\nimport java.util\nimport java.util.Locale\nimport java.util.ServiceLoader\nimport java.util.UUID\nimport java.util.concurrent.TimeUnit\nimport java.util.function.Supplier\n\nimport scala.collection.JavaConverters._\nimport scala.language.implicitConversions\n\nclass Neo4jOptions(private val options: java.util.Map[String, String]) extends Serializable with Logging {\n\n import Neo4jOptions._\n import QueryType._\n\n def asMap() = new util.HashMap[String, String](options)\n\n private def getRequiredParameter(parameter: String): String = {\n if (!options.containsKey(parameter) || options.get(parameter).isEmpty) {\n throw new IllegalArgumentException(s\"Parameter '$parameter' is required\")\n }\n\n options.get(parameter)\n }\n\n private def getParameter(parameter: String, defaultValue: String = \"\"): String =\n getParameterOption(parameter).getOrElse(defaultValue)\n\n private def getParameterOption(parameter: String): Option[String] =\n Some(options.get(parameter))\n .flatMap(Option(_)) // to turn null into None\n .map(_.trim)\n\n private def getAuthenticationParameters: Map[String, String] = {\n val authType = getParameter(AUTH_TYPE, DEFAULT_AUTH_TYPE)\n val authNamespace = s\"$AUTH.$authType\"\n val providedParameters = options.asScala\n .filterKeys(_.startsWith(authNamespace))\n .map(t => (t._1.substring(authNamespace.length + 1), t._2))\n .toMap\n DEFAULT_AUTH_PARAMETERS ++ providedParameters\n }\n\n val saveMode: String = getParameter(SAVE_MODE, DEFAULT_SAVE_MODE.toString)\n\n val pushdownFiltersEnabled: Boolean =\n getParameter(PUSHDOWN_FILTERS_ENABLED, DEFAULT_PUSHDOWN_FILTERS_ENABLED.toString).toBoolean\n\n val pushdownColumnsEnabled: Boolean =\n getParameter(PUSHDOWN_COLUMNS_ENABLED, DEFAULT_PUSHDOWN_COLUMNS_ENABLED.toString).toBoolean\n\n val pushdownAggregateEnabled: Boolean =\n getParameter(PUSHDOWN_AGGREGATE_ENABLED, DEFAULT_PUSHDOWN_AGGREGATE_ENABLED.toString).toBoolean\n\n val pushdownLimitEnabled: Boolean =\n getParameter(PUSHDOWN_LIMIT_ENABLED, DEFAULT_PUSHDOWN_LIMIT_ENABLED.toString).toBoolean\n\n val pushdownTopNEnabled: Boolean =\n getParameter(PUSHDOWN_TOPN_ENABLED, DEFAULT_PUSHDOWN_TOPN_ENABLED.toString).toBoolean\n\n val schemaMetadata: Neo4jSchemaMetadata = initSchemaMetadata\n\n private def initSchemaMetadata = {\n val deprecatedSchemaOptimization = OptimizationType\n .withCaseInsensitiveName(getParameter(SCHEMA_OPTIMIZATION_TYPE, DEFAULT_OPTIMIZATION_TYPE.toString).toUpperCase)\n if (deprecatedSchemaOptimization != OptimizationType.NONE) {\n logWarning(\n s\"\"\"\n |Option `$SCHEMA_OPTIMIZATION_TYPE` is deprecated and will be removed in future implementations,\n |please move to one of the following depending on your use case:\n |- `$SCHEMA_OPTIMIZATION_NODE_KEY`\n |- `$SCHEMA_OPTIMIZATION_RELATIONSHIP_KEY`\n |\"\"\".stripMargin\n )\n }\n\n val nodeConstr: ConstraintsOptimizationType.Value = ConstraintsOptimizationType\n .withCaseInsensitiveName(getParameter(\n SCHEMA_OPTIMIZATION_NODE_KEY,\n DEFAULT_SCHEMA_OPTIMIZATION_NODE_KEY.toString\n ).trim)\n val relConstr: ConstraintsOptimizationType.Value = ConstraintsOptimizationType\n .withCaseInsensitiveName(getParameter(\n SCHEMA_OPTIMIZATION_RELATIONSHIP_KEY,\n DEFAULT_SCHEMA_OPTIMIZATION_RELATIONSHIP_KEY.toString\n ).trim)\n val schemaConstraints = getParameter(SCHEMA_OPTIMIZATION, DEFAULT_SCHEMA_OPTIMIZATION.toString)\n .split(\",\")\n .map(_.trim)\n .map(SchemaConstraintsOptimizationType.withCaseInsensitiveName)\n .toSet\n Neo4jSchemaMetadata(\n getParameter(SCHEMA_FLATTEN_LIMIT, DEFAULT_SCHEMA_FLATTEN_LIMIT.toString).toInt,\n SchemaStrategy.withCaseInsensitiveName(getParameter(\n SCHEMA_STRATEGY,\n DEFAULT_SCHEMA_STRATEGY.toString\n ).toUpperCase),\n deprecatedSchemaOptimization,\n Neo4jSchemaOptimizations(nodeConstr, relConstr, schemaConstraints),\n getParameter(SCHEMA_MAP_GROUP_DUPLICATE_KEYS, DEFAULT_MAP_GROUP_DUPLICATE_KEYS.toString).toBoolean\n )\n }\n\n val indexAwait = getParameter(INDEX_AWAIT_TIMEOUT_SEC, DEFAULT_INDEX_AWAIT_TIMEOUT_SEC.toString).toInt\n\n val query: Neo4jQueryOptions = (\n getParameter(QUERY.toString.toLowerCase),\n getParameter(LABELS.toString.toLowerCase),\n getParameter(RELATIONSHIP.toString.toLowerCase()),\n getParameter(GDS.toString.toLowerCase())\n ) match {\n case (query, \"\", \"\", \"\") => Neo4jQueryOptions(QUERY, query)\n case (\"\", label, \"\", \"\") => {\n val parsed = if (label.trim.startsWith(\":\")) label.substring(1) else label\n Neo4jQueryOptions(LABELS, parsed)\n }\n case (\"\", \"\", relationship, \"\") => Neo4jQueryOptions(RELATIONSHIP, relationship)\n case (\"\", \"\", \"\", gds) => Neo4jQueryOptions(GDS, gds)\n case _ => throw new IllegalArgumentException(\n s\"You need to specify just one of these options: ${\n QueryType.values.toSeq.map(value => s\"'${value.toString.toLowerCase()}'\")\n .sorted.mkString(\", \")\n }\"\n )\n }\n\n val connection: Neo4jDriverOptions = Neo4jDriverOptions(\n getRequiredParameter(URL),\n getParameter(AUTH_TYPE, DEFAULT_AUTH_TYPE),\n getAuthenticationParameters,\n getParameter(ENCRYPTION_ENABLED, DEFAULT_ENCRYPTION_ENABLED.toString).toBoolean,\n Option(getParameter(ENCRYPTION_TRUST_STRATEGY, null)),\n getParameter(ENCRYPTION_CA_CERTIFICATE_PATH, DEFAULT_EMPTY),\n getParameter(CONNECTION_MAX_LIFETIME_MSECS, DEFAULT_CONNECTION_MAX_LIFETIME_MSECS.toString).toInt,\n getParameter(CONNECTION_ACQUISITION_TIMEOUT_MSECS, DEFAULT_TIMEOUT.toString).toInt,\n getParameter(\n CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS,\n DEFAULT_CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS.toString\n ).toInt,\n getParameter(CONNECTION_TIMEOUT_MSECS, DEFAULT_TIMEOUT.toString).toInt\n )\n\n val session: Neo4jSessionOptions = Neo4jSessionOptions(\n getParameter(DATABASE, DEFAULT_EMPTY),\n AccessMode.valueOf(getParameter(ACCESS_MODE, DEFAULT_ACCESS_MODE.toString).toUpperCase())\n )\n\n val nodeMetadata: Neo4jNodeMetadata = initNeo4jNodeMetadata()\n\n val legacyTypeConversionEnabled: Boolean = getParameter(\n TYPE_CONVERSION,\n DEFAULT_TYPE_CONVERSION\n ).toLowerCase(Locale.ROOT).equals(\"legacy\")\n\n private def mapPropsString(strOpt: Option[String]): Option[Map[String, String]] = strOpt.map(str =>\n str.split(\",\")\n .map(_.trim)\n .filter(_.nonEmpty)\n .map(s => {\n val keys = if (s.startsWith(\"`\")) {\n val pattern = \"`[^`]+`\".r\n val groups = pattern findAllIn s\n groups\n .map(_.replaceAll(\"`\", \"\"))\n .toArray\n } else {\n s.split(\":\")\n }\n if (keys.length == 2) {\n (keys(0), keys(1))\n } else {\n (keys(0), keys(0))\n }\n })\n .toMap\n )\n\n private def initNeo4jNodeMetadata(\n nodeKeysString: String = getParameter(NODE_KEYS, \"\"),\n labelsString: String = query.value,\n nodePropsString: String = \"\",\n skipNullKeys: Boolean = getParameter(NODE_KEYS_SKIP_NULLS, \"false\").toBoolean\n ): Neo4jNodeMetadata = {\n\n val nodeKeys = mapPropsString(Some(nodeKeysString)).getOrElse(Map.empty[String, String])\n val nodeProps = mapPropsString(Some(nodePropsString)).getOrElse(Map.empty[String, String])\n\n val labels = labelsString\n .split(\":\")\n .map(_.trim)\n .filter(_.nonEmpty)\n Neo4jNodeMetadata(labels, nodeKeys, nodeProps, skipNullKeys)\n }\n\n val transactionSettings: Neo4jTransactionSettings = initNeo4jTransactionSettings()\n\n val script: Array[String] = getParameter(SCRIPT)\n .split(\";\")\n .map(_.trim)\n .filterNot(_.isEmpty)\n\n private def initNeo4jTransactionSettings(): Neo4jTransactionSettings = {\n val retries = getParameter(TRANSACTION_RETRIES, DEFAULT_TRANSACTION_RETRIES.toString).toInt\n val failOnTransactionCodes = getParameter(TRANSACTION_CODES_FAIL, DEFAULT_EMPTY)\n .split(\",\")\n .map(_.trim)\n .filter(_.nonEmpty)\n .toSet\n val batchSize = getParameter(BATCH_SIZE, DEFAULT_BATCH_SIZE.toString).toInt\n val retryTimeout = getParameter(TRANSACTION_RETRY_TIMEOUT, DEFAULT_TRANSACTION_RETRY_TIMEOUT.toString).toInt\n Neo4jTransactionSettings(retries, failOnTransactionCodes, batchSize, retryTimeout)\n }\n\n val relationshipMetadata: Neo4jRelationshipMetadata = initNeo4jRelationshipMetadata()\n\n private def initNeo4jRelationshipMetadata(): Neo4jRelationshipMetadata = {\n val source = initNeo4jNodeMetadata(\n getParameter(RELATIONSHIP_SOURCE_NODE_KEYS, \"\"),\n getParameter(RELATIONSHIP_SOURCE_LABELS, \"\"),\n getParameter(RELATIONSHIP_SOURCE_NODE_PROPS, \"\"),\n getParameter(RELATIONSHIP_SOURCE_NODE_KEYS_SKIP_NULLS, \"false\").toBoolean\n )\n\n val target = initNeo4jNodeMetadata(\n getParameter(RELATIONSHIP_TARGET_NODE_KEYS, \"\"),\n getParameter(RELATIONSHIP_TARGET_LABELS, \"\"),\n getParameter(RELATIONSHIP_TARGET_NODE_PROPS, \"\"),\n getParameter(RELATIONSHIP_TARGET_NODE_KEYS_SKIP_NULLS, \"false\").toBoolean\n )\n\n val nodeMap = getParameter(RELATIONSHIP_NODES_MAP, DEFAULT_RELATIONSHIP_NODES_MAP.toString).toBoolean\n\n val relProps = mapPropsString(getParameterOption(RELATIONSHIP_PROPERTIES))\n\n val writeStrategy = RelationshipSaveStrategy.withCaseInsensitiveName(getParameter(\n RELATIONSHIP_SAVE_STRATEGY,\n DEFAULT_RELATIONSHIP_SAVE_STRATEGY.toString\n ).toUpperCase)\n val sourceSaveMode = NodeSaveMode.withCaseInsensitiveName(getParameter(\n RELATIONSHIP_SOURCE_SAVE_MODE,\n DEFAULT_RELATIONSHIP_SOURCE_SAVE_MODE.toString\n ))\n val targetSaveMode = NodeSaveMode.withCaseInsensitiveName(getParameter(\n RELATIONSHIP_TARGET_SAVE_MODE,\n DEFAULT_RELATIONSHIP_TARGET_SAVE_MODE.toString\n ))\n\n val relationshipKeys = mapPropsString(getParameterOption(RELATIONSHIP_KEYS)).getOrElse(Map.empty)\n\n Neo4jRelationshipMetadata(\n source,\n target,\n sourceSaveMode,\n targetSaveMode,\n relProps,\n query.value,\n nodeMap,\n writeStrategy,\n relationshipKeys,\n getParameter(RELATIONSHIP_KEYS_SKIP_NULLS, \"false\").toBoolean\n )\n }\n\n private def initNeo4jQueryMetadata(): Neo4jQueryMetadata = Neo4jQueryMetadata(\n query.value.trim,\n getParameter(QUERY_COUNT, \"\").trim\n )\n\n val queryMetadata: Neo4jQueryMetadata = initNeo4jQueryMetadata()\n\n private def initNeo4jGdsMetadata(): Neo4jGdsMetadata = Neo4jGdsMetadata(\n options.asScala\n .filterKeys(k => k.startsWith(\"gds.\"))\n .map(t => (t._1.substring(\"gds.\".length), t._2))\n .toMap\n .toNestedJavaMap\n )\n\n val gdsMetadata: Neo4jGdsMetadata = initNeo4jGdsMetadata()\n\n val partitions: Int = getParameter(PARTITIONS, DEFAULT_PARTITIONS.toString).toInt\n\n val streamingOrderBy: String = getParameter(ORDER_BY, getParameter(STREAMING_PROPERTY_NAME))\n\n val apocConfig: Neo4jApocConfig = Neo4jApocConfig(options.asScala\n .filterKeys(_.startsWith(\"apoc.\"))\n .mapValues(Neo4jUtil.mapper.readValue(_, classOf[java.util.Map[String, AnyRef]]).asScala)\n .toMap)\n\n def getTableName: String = query.queryType match {\n case QueryType.LABELS => s\"table_${nodeMetadata.labels.mkString(\"-\")}\"\n case QueryType.RELATIONSHIP => s\"table_${relationshipMetadata.source.labels.mkString(\"-\")}\" +\n s\"_${relationshipMetadata.relationshipType}\" +\n s\"_${relationshipMetadata.target.labels.mkString(\"-\")}\"\n case _ => s\"table_query_${UUID.randomUUID()}\"\n }\n\n val streamingOptions: Neo4jStreamingOptions = Neo4jStreamingOptions(\n getParameter(STREAMING_PROPERTY_NAME),\n StreamingFrom.withCaseInsensitiveName(getParameter(STREAMING_FROM, DEFAULT_STREAMING_FROM.toString)),\n getParameter(STREAMING_QUERY_OFFSET)\n )\n\n def toNeo4jTransactionConfig: TransactionConfig = {\n val timeout = getParameter(TRANSACTION_TIMEOUT_MSECS, DEFAULT_TRANSACTION_TIMEOUT)\n\n val builder = TransactionConfig.builder()\n if (timeout != null) {\n val duration = Duration.ofMillis(timeout.toInt)\n builder.withTimeout(duration)\n }\n\n builder.build()\n }\n\n}\n\ncase class Neo4jStreamingOptions(\n propertyName: String,\n from: StreamingFrom.Value,\n queryOffset: String\n)\n\ncase class Neo4jApocConfig(procedureConfigMap: Map[String, AnyRef])\n\ncase class Neo4jSchemaOptimizations(\n nodeConstraint: ConstraintsOptimizationType.Value,\n relConstraint: ConstraintsOptimizationType.Value,\n schemaConstraints: Set[SchemaConstraintsOptimizationType.Value]\n)\n\ncase class Neo4jSchemaMetadata(\n flattenLimit: Int,\n strategy: SchemaStrategy.Value,\n optimizationType: OptimizationType.Value,\n optimization: Neo4jSchemaOptimizations,\n mapGroupDuplicateKeys: Boolean\n)\n\ncase class Neo4jTransactionSettings(\n retries: Int,\n failOnTransactionCodes: Set[String],\n batchSize: Int,\n retryTimeout: Long\n) {\n\n def shouldFailOn(exception: Throwable): Boolean = {\n exception match {\n case e: Neo4jException => failOnTransactionCodes.contains(e.code())\n case _ => false\n }\n }\n\n}\n\ncase class Neo4jNodeMetadata(\n labels: Seq[String],\n nodeKeys: Map[String, String],\n properties: Map[String, String],\n skipNullKeys: Boolean = false\n) {\n def includesProperty(name: String): Boolean = nodeKeys.contains(name) || properties.contains(name)\n}\n\ncase class Neo4jRelationshipMetadata(\n source: Neo4jNodeMetadata,\n target: Neo4jNodeMetadata,\n sourceSaveMode: NodeSaveMode.Value,\n targetSaveMode: NodeSaveMode.Value,\n properties: Option[Map[String, String]],\n relationshipType: String,\n nodeMap: Boolean,\n saveStrategy: RelationshipSaveStrategy.Value,\n relationshipKeys: Map[String, String],\n skipNullKeys: Boolean = false\n)\n\ncase class Neo4jQueryMetadata(query: String, queryCount: String)\n\ncase class Neo4jGdsMetadata(parameters: util.Map[String, Any])\n\ncase class Neo4jQueryOptions(queryType: QueryType.Value, value: String)\n\ncase class Neo4jSessionOptions(database: String, accessMode: AccessMode = AccessMode.READ) {\n\n def toNeo4jSession(): SessionConfig = {\n val builder = SessionConfig.builder()\n .withDefaultAccessMode(accessMode)\n\n if (database != null && database != \"\") {\n builder.withDatabase(database)\n }\n\n builder.build()\n }\n}\n\ncase class Neo4jDriverOptions(\n url: String,\n auth: String,\n authParameters: Map[String, String],\n encryption: Boolean,\n trustStrategy: Option[String],\n certificatePath: String,\n lifetime: Int,\n acquisitionTimeout: Int,\n livenessCheckTimeout: Int,\n connectionTimeout: Int\n) extends Serializable {\n\n def createDriver(): Driver = {\n val (url, _) = connectionUrls\n ReAuthDriverFactory.driver(url, createAuthTokenSupplier, toDriverConfig)\n }\n\n private def toDriverConfig: Config = {\n val builder = Config.builder()\n .withUserAgent(s\"neo4j-${Neo4jUtil.connectorEnv}-connector/${Neo4jUtil.connectorVersion}\")\n .withLogging(Logging.slf4j())\n\n if (lifetime > -1) builder.withMaxConnectionLifetime(lifetime, TimeUnit.MILLISECONDS)\n if (acquisitionTimeout > -1) builder.withConnectionAcquisitionTimeout(acquisitionTimeout, TimeUnit.MILLISECONDS)\n if (livenessCheckTimeout > -1)\n builder.withConnectionLivenessCheckTimeout(livenessCheckTimeout, TimeUnit.MILLISECONDS)\n if (connectionTimeout > -1) builder.withConnectionTimeout(connectionTimeout, TimeUnit.MILLISECONDS)\n\n val (primaryUrl, resolvers) = connectionUrls\n\n primaryUrl.getScheme match {\n case \"neo4j+s\" | \"neo4j+ssc\" | \"bolt+s\" | \"bolt+ssc\" => ()\n case _ => {\n if (!encryption) {\n builder.withoutEncryption()\n } else {\n builder.withEncryption()\n }\n trustStrategy\n .map(Config.TrustStrategy.Strategy.valueOf)\n .map {\n case TrustStrategy.Strategy.TRUST_ALL_CERTIFICATES => TrustStrategy.trustAllCertificates()\n case TrustStrategy.Strategy.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES => TrustStrategy.trustSystemCertificates()\n case TrustStrategy.Strategy.TRUST_CUSTOM_CA_SIGNED_CERTIFICATES =>\n TrustStrategy.trustCustomCertificateSignedBy(new File(certificatePath))\n }.foreach(builder.withTrustStrategy)\n }\n }\n\n if (resolvers.nonEmpty) {\n builder.withResolver(_ => resolvers.asJava)\n }\n\n builder.build()\n }\n\n // public only for testing purposes\n @TestOnly\n def connectionUrls: (URI, Set[ServerAddress]) = {\n val urls = url.split(\",\").toList\n val resolved = urls\n .drop(1)\n .map(_.trim)\n .map(URI.create)\n .map(uri => ServerAddress.of(uri.getHost, if (uri.getPort > -1) uri.getPort else 7687))\n .toSet\n (URI.create(urls.head.trim), resolved)\n }\n\n private def createAuthTokenSupplier: Supplier[AuthenticationToken] = {\n if (auth == null || auth.isEmpty) {\n throw new IllegalArgumentException(s\"Authentication type name is required\")\n }\n val supplierFactories = ServiceLoader.load(\n classOf[AuthenticationTokenSupplierFactory],\n getClass.getClassLoader\n ).iterator()\n .asScala\n .toList\n\n val filteredSupplierFactories = supplierFactories.filter(s => s.getName != null && s.getName.equalsIgnoreCase(auth))\n\n if (filteredSupplierFactories.isEmpty) {\n throw new IllegalArgumentException(\n s\"Authentication method '$auth' is not supported. Supported authentication methods are: ${supplierFactories.map(_.getName).mkString(\", \")}\"\n )\n }\n if (filteredSupplierFactories.size > 1) {\n throw new IllegalArgumentException(\n s\"Multiple implementation for authentication type '$auth' are found\"\n )\n }\n\n val username = authParameters.get(\"username\")\n val password = authParameters.get(\"password\")\n filteredSupplierFactories.head.create(username.orNull, password.orNull, authParameters.asJava)\n }\n\n}\n\nobject Neo4jOptions {\n\n // connection options\n val URL = \"url\"\n\n // auth\n val AUTH = \"authentication\"\n val AUTH_TYPE = \"authentication.type\"\n\n // driver\n val ENCRYPTION_ENABLED = \"encryption.enabled\"\n val ENCRYPTION_TRUST_STRATEGY = \"encryption.trust.strategy\"\n val ENCRYPTION_CA_CERTIFICATE_PATH = \"encryption.ca.certificate.path\"\n val CONNECTION_MAX_LIFETIME_MSECS = \"connection.max.lifetime.msecs\"\n val CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS = \"connection.liveness.timeout.msecs\"\n val CONNECTION_ACQUISITION_TIMEOUT_MSECS = \"connection.acquisition.timeout.msecs\"\n val CONNECTION_TIMEOUT_MSECS = \"connection.timeout.msecs\"\n val TRANSACTION_TIMEOUT_MSECS = \"db.transaction.timeout\"\n\n // session options\n val DATABASE = \"database\"\n val ACCESS_MODE = \"access.mode\"\n val SAVE_MODE = \"save.mode\"\n\n val PUSHDOWN_FILTERS_ENABLED = \"pushdown.filters.enabled\"\n val PUSHDOWN_COLUMNS_ENABLED = \"pushdown.columns.enabled\"\n val PUSHDOWN_AGGREGATE_ENABLED = \"pushdown.aggregate.enabled\"\n val PUSHDOWN_LIMIT_ENABLED = \"pushdown.limit.enabled\"\n val PUSHDOWN_TOPN_ENABLED = \"pushdown.topN.enabled\"\n\n // schema options\n val SCHEMA_STRATEGY = \"schema.strategy\"\n val SCHEMA_FLATTEN_LIMIT = \"schema.flatten.limit\"\n // deprecated in favor of...\n val SCHEMA_OPTIMIZATION_TYPE = \"schema.optimization.type\"\n // ...these options\n val SCHEMA_OPTIMIZATION = \"schema.optimization\"\n val SCHEMA_OPTIMIZATION_NODE_KEY = \"schema.optimization.node.keys\"\n val SCHEMA_OPTIMIZATION_RELATIONSHIP_KEY = \"schema.optimization.relationship.keys\"\n // map aggregation\n val SCHEMA_MAP_GROUP_DUPLICATE_KEYS = \"schema.map.group.duplicate.keys\"\n\n // index options\n val INDEX_AWAIT_TIMEOUT_SEC = \"index.await.timeout\"\n\n // partitions\n val PARTITIONS = \"partitions\"\n\n // orderBy\n val ORDER_BY = \"orderBy\"\n\n // skip.nulls postfix\n val SKIP_NULLS = \"skip.nulls\"\n\n // Node Metadata\n val NODE_KEYS = \"node.keys\"\n val NODE_KEYS_SKIP_NULLS = s\"${NODE_KEYS}.${SKIP_NULLS}\"\n val NODE_PROPS = \"node.properties\"\n\n val BATCH_SIZE = \"batch.size\"\n val SUPPORTED_SAVE_MODES = Seq(SaveMode.Overwrite, SaveMode.ErrorIfExists, SaveMode.Append)\n\n // Relationship Metadata\n val RELATIONSHIP_SOURCE_LABELS =\n s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.source.${QueryType.LABELS.toString.toLowerCase}\"\n val RELATIONSHIP_SOURCE_NODE_KEYS = s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.source.$NODE_KEYS\"\n val RELATIONSHIP_SOURCE_NODE_KEYS_SKIP_NULLS = s\"${RELATIONSHIP_SOURCE_NODE_KEYS}.${SKIP_NULLS}\"\n val RELATIONSHIP_SOURCE_NODE_PROPS = s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.source.$NODE_PROPS\"\n val RELATIONSHIP_SOURCE_SAVE_MODE = s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.source.$SAVE_MODE\"\n\n val RELATIONSHIP_TARGET_LABELS =\n s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.target.${QueryType.LABELS.toString.toLowerCase}\"\n val RELATIONSHIP_TARGET_NODE_KEYS = s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.target.$NODE_KEYS\"\n val RELATIONSHIP_TARGET_NODE_KEYS_SKIP_NULLS = s\"${RELATIONSHIP_TARGET_NODE_KEYS}.${SKIP_NULLS}\"\n val RELATIONSHIP_TARGET_NODE_PROPS = s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.target.$NODE_PROPS\"\n val RELATIONSHIP_TARGET_SAVE_MODE = s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.target.$SAVE_MODE\"\n val RELATIONSHIP_PROPERTIES = s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.properties\"\n val RELATIONSHIP_NODES_MAP = s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.nodes.map\"\n val RELATIONSHIP_SAVE_STRATEGY = s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.save.strategy\"\n val RELATIONSHIP_KEYS = s\"${QueryType.RELATIONSHIP.toString.toLowerCase}.keys\"\n val RELATIONSHIP_KEYS_SKIP_NULLS = s\"${RELATIONSHIP_KEYS}.${SKIP_NULLS}\"\n\n // Query metadata\n val QUERY_COUNT = \"query.count\"\n\n // Transaction Metadata\n val TRANSACTION_RETRIES = \"transaction.retries\"\n val TRANSACTION_RETRY_TIMEOUT = \"transaction.retry.timeout\"\n val TRANSACTION_CODES_FAIL = \"transaction.codes.fail\"\n\n // Streaming\n val STREAMING_PROPERTY_NAME = \"streaming.property.name\"\n val STREAMING_FROM = \"streaming.from\"\n val STREAMING_QUERY_OFFSET = \"streaming.query.offset\"\n\n val SCRIPT = \"script\"\n\n // Data conversion\n val TYPE_CONVERSION = \"type.conversion\"\n\n // defaults\n val DEFAULT_EMPTY = \"\"\n val DEFAULT_TIMEOUT: Int = -1\n val DEFAULT_ACCESS_MODE = AccessMode.READ\n val DEFAULT_AUTH_TYPE = \"basic\"\n val DEFAULT_ENCRYPTION_ENABLED = false\n val DEFAULT_SCHEMA_FLATTEN_LIMIT = 10\n val DEFAULT_BATCH_SIZE = 5000\n val DEFAULT_TRANSACTION_RETRIES = 3\n val DEFAULT_TRANSACTION_RETRY_TIMEOUT = 0\n val DEFAULT_TRANSACTION_TIMEOUT = null\n val DEFAULT_RELATIONSHIP_NODES_MAP = false\n val DEFAULT_SCHEMA_STRATEGY = SchemaStrategy.SAMPLE\n val DEFAULT_SCHEMA_OPTIMIZATION_NODE_KEY = ConstraintsOptimizationType.NONE\n val DEFAULT_SCHEMA_OPTIMIZATION_RELATIONSHIP_KEY = ConstraintsOptimizationType.NONE\n val DEFAULT_SCHEMA_OPTIMIZATION = SchemaConstraintsOptimizationType.NONE\n val DEFAULT_RELATIONSHIP_SAVE_STRATEGY: RelationshipSaveStrategy.Value = RelationshipSaveStrategy.NATIVE\n val DEFAULT_RELATIONSHIP_SOURCE_SAVE_MODE: NodeSaveMode.Value = NodeSaveMode.Match\n val DEFAULT_RELATIONSHIP_TARGET_SAVE_MODE: NodeSaveMode.Value = NodeSaveMode.Match\n val DEFAULT_PUSHDOWN_FILTERS_ENABLED = true\n val DEFAULT_PUSHDOWN_COLUMNS_ENABLED = true\n val DEFAULT_PUSHDOWN_AGGREGATE_ENABLED = true\n val DEFAULT_PUSHDOWN_LIMIT_ENABLED = true\n val DEFAULT_PUSHDOWN_TOPN_ENABLED = true\n val DEFAULT_PARTITIONS = 1\n val DEFAULT_OPTIMIZATION_TYPE = OptimizationType.NONE\n val DEFAULT_SAVE_MODE = SaveMode.Overwrite\n val DEFAULT_STREAMING_FROM = StreamingFrom.NOW\n\n // Default values optimizations for Aura please look at: https://aura.support.neo4j.com/hc/en-us/articles/1500002493281-Neo4j-Java-driver-settings-for-Aura\n val DEFAULT_CONNECTION_MAX_LIFETIME_MSECS = Duration.ofMinutes(8).toMillis\n val DEFAULT_CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS = Duration.ofMinutes(2).toMillis\n\n val DEFAULT_MAP_GROUP_DUPLICATE_KEYS = false\n\n val DEFAULT_INDEX_AWAIT_TIMEOUT_SEC = 300\n\n var DEFAULT_AUTH_PARAMETERS: Map[String, String] =\n Seq(\"username\", \"password\", \"ticket\", \"principal\", \"credentials\", \"realm\", \"scheme\", \"token\")\n .map(name => name -> DEFAULT_EMPTY).toMap\n\n private val DEFAULT_TYPE_CONVERSION = \"default\"\n\n def fromSession(sparkSession: Option[SparkSession], options: java.util.Map[String, String]): Neo4jOptions = {\n val sessionLevelOptions = sparkSession\n .map {\n _.conf\n .getAll\n .filterKeys(k => k.startsWith(\"neo4j.\"))\n .map { elem => (elem._1.substring(\"neo4j.\".length), elem._2) }\n .toMap\n }\n .getOrElse(Map.empty)\n\n new Neo4jOptions((sessionLevelOptions ++ options.asScala).asJava)\n }\n}\n\nclass CaseInsensitiveEnumeration extends Enumeration {\n\n def withCaseInsensitiveName(s: String): Value = {\n values.find(_.toString.toLowerCase() == s.toLowerCase).getOrElse(\n throw new NoSuchElementException(s\"No value found for '$s'\")\n )\n }\n}\n\nobject StreamingFrom extends CaseInsensitiveEnumeration {\n val ALL, NOW = Value\n\n class StreamingFromValue(value: Value) {\n\n def value(): Long = value match {\n case ALL => -1L\n case NOW => System.currentTimeMillis()\n }\n }\n\n implicit def valToStreamingFromValue(value: Value): StreamingFromValue = new StreamingFromValue(value)\n}\n\nobject StorageType extends CaseInsensitiveEnumeration {\n val NEO4J, SPARK = Value\n}\n\nobject QueryType extends CaseInsensitiveEnumeration {\n val QUERY, LABELS, RELATIONSHIP, GDS = Value\n}\n\nobject RelationshipSaveStrategy extends CaseInsensitiveEnumeration {\n val NATIVE, KEYS = Value\n}\n\nobject NodeSaveMode extends CaseInsensitiveEnumeration {\n val Overwrite, ErrorIfExists, Match, Append = Value\n\n def fromSaveMode(saveMode: SaveMode): Value = {\n saveMode match {\n case SaveMode.Overwrite => Overwrite\n case SaveMode.ErrorIfExists => ErrorIfExists\n case SaveMode.Append => Append\n case _ => throw new IllegalArgumentException(s\"SaveMode $saveMode not supported\")\n }\n }\n}\n\nobject SchemaStrategy extends CaseInsensitiveEnumeration {\n val STRING, SAMPLE = Value\n}\n\nobject OptimizationType extends CaseInsensitiveEnumeration {\n val INDEX, NODE_CONSTRAINTS, NONE = Value\n}\n\nobject ConstraintsOptimizationType extends CaseInsensitiveEnumeration {\n val KEY, UNIQUE, NONE = Value\n}\n\nobject SchemaConstraintsOptimizationType extends CaseInsensitiveEnumeration {\n val TYPE, EXISTS, NONE = Value\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/util/Neo4jUtil.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport com.fasterxml.jackson.core.JsonGenerator\nimport com.fasterxml.jackson.databind.JsonSerializer\nimport com.fasterxml.jackson.databind.ObjectMapper\nimport com.fasterxml.jackson.databind.SerializerProvider\nimport com.fasterxml.jackson.databind.module.SimpleModule\nimport org.apache.spark.sql.sources._\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.cypherdsl.core._\nimport org.neo4j.driver.Session\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.internal.retry.ExponentialBackoffRetryLogic\nimport org.neo4j.driver.types.Entity\nimport org.neo4j.driver.types.Path\nimport org.neo4j.spark.service.SchemaService\nimport org.neo4j.spark.util.Neo4jImplicits.EntityImplicits\nimport org.neo4j.spark.util.Neo4jImplicits._\nimport org.slf4j.Logger\n\nimport java.time.temporal.Temporal\nimport java.util.Properties\n\nimport scala.annotation.tailrec\n\nobject Neo4jUtil {\n\n val NODE_ALIAS = \"n\"\n private val INTERNAL_ID_FIELD_NAME = \"id\"\n val INTERNAL_ID_FIELD = s\"<${INTERNAL_ID_FIELD_NAME}>\"\n private val INTERNAL_LABELS_FIELD_NAME = \"labels\"\n val INTERNAL_LABELS_FIELD = s\"<${INTERNAL_LABELS_FIELD_NAME}>\"\n val INTERNAL_REL_ID_FIELD = s\"\"\n val INTERNAL_REL_TYPE_FIELD = \"\"\n val RELATIONSHIP_SOURCE_ALIAS = \"source\"\n val RELATIONSHIP_TARGET_ALIAS = \"target\"\n val INTERNAL_REL_SOURCE_ID_FIELD = s\"<${RELATIONSHIP_SOURCE_ALIAS}.${INTERNAL_ID_FIELD_NAME}>\"\n val INTERNAL_REL_TARGET_ID_FIELD = s\"<${RELATIONSHIP_TARGET_ALIAS}.${INTERNAL_ID_FIELD_NAME}>\"\n val INTERNAL_REL_SOURCE_LABELS_FIELD = s\"<${RELATIONSHIP_SOURCE_ALIAS}.${INTERNAL_LABELS_FIELD_NAME}>\"\n val INTERNAL_REL_TARGET_LABELS_FIELD = s\"<${RELATIONSHIP_TARGET_ALIAS}.${INTERNAL_LABELS_FIELD_NAME}>\"\n val RELATIONSHIP_ALIAS = \"rel\"\n\n private val properties = new Properties()\n properties.load(Thread.currentThread().getContextClassLoader.getResourceAsStream(\"neo4j-spark-connector.properties\"))\n\n def closeSafely(autoCloseable: AutoCloseable, logger: Logger = null): Unit = {\n try {\n autoCloseable match {\n case s: Session => if (s.isOpen) s.close()\n case t: Transaction => if (t.isOpen) t.close()\n case null => ()\n case _ => autoCloseable.close()\n }\n } catch {\n case t: Throwable => if (logger != null) logger\n .warn(s\"Cannot close ${autoCloseable.getClass.getSimpleName} because of the following exception:\", t)\n }\n }\n\n val mapper = new ObjectMapper()\n private val module = new SimpleModule(\"Neo4jApocSerializer\")\n\n module.addSerializer(\n classOf[Path],\n new JsonSerializer[Path]() {\n\n override def serialize(path: Path, jsonGenerator: JsonGenerator, serializerProvider: SerializerProvider): Unit =\n jsonGenerator.writeString(path.toString)\n }\n )\n\n module.addSerializer(\n classOf[Entity],\n new JsonSerializer[Entity]() {\n\n override def serialize(\n entity: Entity,\n jsonGenerator: JsonGenerator,\n serializerProvider: SerializerProvider\n ): Unit = jsonGenerator.writeObject(entity.toMap)\n }\n )\n\n module.addSerializer(\n classOf[Temporal],\n new JsonSerializer[Temporal]() {\n\n override def serialize(\n entity: Temporal,\n jsonGenerator: JsonGenerator,\n serializerProvider: SerializerProvider\n ): Unit = jsonGenerator.writeRaw(entity.toString)\n }\n )\n mapper.registerModule(module)\n\n def isLong(str: String): Boolean = {\n if (str == null) {\n false\n } else {\n try {\n str.trim.toLong\n true\n } catch {\n case _: NumberFormatException => false\n }\n }\n }\n\n def connectorVersion: String = properties.getOrDefault(\"version\", \"UNKNOWN\").toString\n\n def connectorEnv: String = Option(System.getProperty(\"neo4j.spark.platform\"))\n .getOrElse(defaultConnectorEnv)\n\n private def defaultConnectorEnv: String = Option(System.getenv(\"DATABRICKS_RUNTIME_VERSION\"))\n .map(_ => \"databricks\")\n .getOrElse(\"spark\")\n\n def getCorrectProperty(container: PropertyContainer, attribute: String): Property = {\n container.property(attribute.split('.'): _*)\n }\n\n def paramsFromFilters(filters: Array[Filter]): Map[String, Any] = {\n filters.flatMap(f => f.flattenFilters).map(_.getAttributeAndValue)\n .filter(_.nonEmpty)\n .map(valAndAtt => valAndAtt.head.toString.unquote() -> toParamValue(valAndAtt(1)))\n .toMap\n }\n\n def toParamValue(value: Any): Any = {\n value match {\n case date: java.sql.Date => date.toString\n case timestamp: java.sql.Timestamp => timestamp.toLocalDateTime\n case _ => value\n }\n }\n\n def valueToCypherExpression(attribute: String, value: Any): Expression = {\n val parameter = Cypher.parameter(attribute.toParameterName(value))\n value match {\n case d: java.sql.Date => Functions.date(parameter)\n case t: java.sql.Timestamp => Functions.localdatetime(parameter)\n case _ => parameter\n }\n }\n\n def mapSparkFiltersToCypher(\n filter: Filter,\n container: PropertyContainer,\n attributeAlias: Option[String] = None\n ): Condition = {\n filter match {\n case eqns: EqualNullSafe =>\n val parameter = valueToCypherExpression(eqns.attribute, eqns.value)\n val property = getCorrectProperty(container, attributeAlias.getOrElse(eqns.attribute))\n property.isNull.and(parameter.isNull)\n .or(property.isEqualTo(parameter))\n case eq: EqualTo =>\n getCorrectProperty(container, attributeAlias.getOrElse(eq.attribute))\n .isEqualTo(valueToCypherExpression(eq.attribute, eq.value))\n case gt: GreaterThan =>\n getCorrectProperty(container, attributeAlias.getOrElse(gt.attribute))\n .gt(valueToCypherExpression(gt.attribute, gt.value))\n case gte: GreaterThanOrEqual =>\n getCorrectProperty(container, attributeAlias.getOrElse(gte.attribute))\n .gte(valueToCypherExpression(gte.attribute, gte.value))\n case lt: LessThan =>\n getCorrectProperty(container, attributeAlias.getOrElse(lt.attribute))\n .lt(valueToCypherExpression(lt.attribute, lt.value))\n case lte: LessThanOrEqual =>\n getCorrectProperty(container, attributeAlias.getOrElse(lte.attribute))\n .lte(valueToCypherExpression(lte.attribute, lte.value))\n case in: In =>\n getCorrectProperty(container, attributeAlias.getOrElse(in.attribute))\n .in(valueToCypherExpression(in.attribute, in.values))\n case startWith: StringStartsWith =>\n getCorrectProperty(container, attributeAlias.getOrElse(startWith.attribute))\n .startsWith(valueToCypherExpression(startWith.attribute, startWith.value))\n case endsWith: StringEndsWith =>\n getCorrectProperty(container, attributeAlias.getOrElse(endsWith.attribute))\n .endsWith(valueToCypherExpression(endsWith.attribute, endsWith.value))\n case contains: StringContains =>\n getCorrectProperty(container, attributeAlias.getOrElse(contains.attribute))\n .contains(valueToCypherExpression(contains.attribute, contains.value))\n case notNull: IsNotNull => getCorrectProperty(container, attributeAlias.getOrElse(notNull.attribute)).isNotNull\n case isNull: IsNull => getCorrectProperty(container, attributeAlias.getOrElse(isNull.attribute)).isNull\n case not: Not => mapSparkFiltersToCypher(not.child, container, attributeAlias).not()\n case filter @ (_: Filter) => throw new IllegalArgumentException(s\"Filter of type `$filter` is not supported.\")\n }\n }\n\n def getStreamingPropertyName(options: Neo4jOptions): String = options.query.queryType match {\n case QueryType.RELATIONSHIP => s\"rel.${options.streamingOptions.propertyName}\"\n case _ => options.streamingOptions.propertyName\n }\n\n def callSchemaService[T](\n neo4j: Neo4j,\n neo4jOptions: Neo4jOptions,\n jobId: String,\n filters: Array[Filter],\n function: SchemaService => T\n ): T = {\n val driverCache = new DriverCache(neo4jOptions.connection)\n val schemaService = new SchemaService(neo4j, neo4jOptions, driverCache, filters)\n var hasError = false\n try {\n function(schemaService)\n } catch {\n case e: Throwable => {\n hasError = true\n throw e\n }\n } finally {\n schemaService.close()\n if (hasError) {\n driverCache.close()\n }\n }\n }\n\n @tailrec\n def isRetryableException(exception: Throwable): Boolean = {\n if (exception == null) {\n false\n } else\n ExponentialBackoffRetryLogic.isRetryable(exception) || isRetryableException(\n exception.getCause\n )\n }\n\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/util/ValidationUtil.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nobject ValidationUtil {\n\n def isNotEmpty(str: String, message: String) = if (str.isEmpty) {\n throw new IllegalArgumentException(message)\n }\n\n def isNotBlank(str: String, message: String) = if (str.trim.isEmpty) {\n throw new IllegalArgumentException(message)\n }\n\n def isBlank(str: String, message: String) = if (!str.trim.isEmpty) {\n throw new IllegalArgumentException(message)\n }\n\n def isNotEmpty(seq: Seq[_], message: String) = if (seq.isEmpty) {\n throw new IllegalArgumentException(message)\n }\n\n def isNotEmpty(map: Map[_, _], message: String) = if (map.isEmpty) {\n throw new IllegalArgumentException(message)\n }\n\n def isTrue(boolean: Boolean, message: String) = if (!boolean) {\n throw new IllegalArgumentException(message)\n }\n\n def isFalse(boolean: Boolean, message: String) = if (boolean) {\n throw new IllegalArgumentException(message)\n }\n\n def isNotValid(message: String) = throw new IllegalArgumentException(message)\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/util/Validations.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport org.apache.spark.internal.Logging\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.SparkSession\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.driver.AccessMode\nimport org.neo4j.driver.summary\nimport org.neo4j.spark.service.Neo4jQueryStrategy\nimport org.neo4j.spark.service.SchemaService\nimport org.neo4j.spark.util\nimport org.neo4j.spark.util.Neo4jImplicits.StructTypeImplicit\n\nimport java.util.Locale\n\nobject Validations {\n def validate(validations: Validation*): Unit = validations.toSet[Validation].foreach(_.validate())\n}\n\ntrait Validation extends Logging {\n def validate(): Unit\n\n def ignoreOption(ignoredOption: String, primaryOption: String): Unit =\n logWarning(s\"Option `$ignoredOption` is not compatible with `$primaryOption` and will be ignored\")\n}\n\ncase class ValidateSchemaOptions(neo4jOptions: Neo4jOptions, schema: StructType) extends Validation {\n\n override def validate(): Unit = {\n val missingFieldsMap = Map(\n Neo4jOptions.NODE_KEYS -> schema.getMissingFields(neo4jOptions.nodeMetadata.nodeKeys.keySet),\n Neo4jOptions.NODE_PROPS -> schema.getMissingFields(neo4jOptions.nodeMetadata.properties.keySet),\n Neo4jOptions.RELATIONSHIP_PROPERTIES -> schema.getMissingFields(\n neo4jOptions.relationshipMetadata.properties.getOrElse(Map.empty).keySet\n ),\n Neo4jOptions.RELATIONSHIP_SOURCE_NODE_PROPS -> schema.getMissingFields(\n neo4jOptions.relationshipMetadata.source.properties.keySet\n ),\n Neo4jOptions.RELATIONSHIP_SOURCE_NODE_KEYS -> schema.getMissingFields(\n neo4jOptions.relationshipMetadata.source.nodeKeys.keySet\n ),\n Neo4jOptions.RELATIONSHIP_TARGET_NODE_PROPS -> schema.getMissingFields(\n neo4jOptions.relationshipMetadata.target.properties.keySet\n ),\n Neo4jOptions.RELATIONSHIP_TARGET_NODE_KEYS -> schema.getMissingFields(\n neo4jOptions.relationshipMetadata.target.nodeKeys.keySet\n )\n )\n\n val optionsWithMissingFields = missingFieldsMap.filter(_._2.nonEmpty)\n\n if (optionsWithMissingFields.nonEmpty) {\n throw new IllegalArgumentException(\n s\"\"\"Write failed due to the following errors:\n |${optionsWithMissingFields.map(field =>\n s\" - Schema is missing ${field._2.mkString(\", \")} from option `${field._1}`\"\n ).mkString(\"\\n\")}\n |\n |The option key and value might be inverted.\"\"\".stripMargin\n )\n }\n }\n}\n\ncase class ValidateSchemaMetadataWrite(neo4jOptions: Neo4jOptions, saveMode: SaveMode) extends Validation {\n\n override def validate(): Unit = {\n val schemaMetadata = neo4jOptions.schemaMetadata\n val hasNodeOptimizations = (schemaMetadata.optimizationType == OptimizationType.NODE_CONSTRAINTS\n || schemaMetadata.optimization.nodeConstraint != ConstraintsOptimizationType.NONE)\n val hasOptimizations = (schemaMetadata.optimizationType != OptimizationType.NONE\n && (schemaMetadata.optimization.nodeConstraint != ConstraintsOptimizationType.NONE\n || schemaMetadata.optimization.relConstraint != ConstraintsOptimizationType.NONE\n || schemaMetadata.optimization.schemaConstraints != Set(SchemaConstraintsOptimizationType.NONE)))\n if (hasOptimizations) {\n throw new IllegalArgumentException(\n s\"\"\"You cannot combine `${Neo4jOptions.SCHEMA_OPTIMIZATION_TYPE}` with:\n |- `${Neo4jOptions.SCHEMA_OPTIMIZATION_NODE_KEY}`\n |- `${Neo4jOptions.SCHEMA_OPTIMIZATION_RELATIONSHIP_KEY}`\n |- `${Neo4jOptions.SCHEMA_OPTIMIZATION}`\n \"\"\"\n )\n }\n neo4jOptions.query.queryType match {\n case QueryType.QUERY => {\n neo4jOptions.schemaMetadata.optimizationType match {\n case OptimizationType.NONE => // are valid\n case _ => ValidationUtil.isNotValid(\n s\"\"\"With Query Type ${neo4jOptions.query.queryType} you can\n |only use `${Neo4jOptions.SCHEMA_OPTIMIZATION_TYPE}`\n |`${OptimizationType.NONE}`\n |\"\"\".stripMargin\n )\n }\n if (hasOptimizations) {\n throw new IllegalArgumentException(\n s\"With Query Type ${neo4jOptions.query.queryType} you cannot define any optimization\"\n )\n }\n }\n case QueryType.LABELS => {\n if (hasNodeOptimizations) {\n ValidationUtil.isTrue(saveMode == SaveMode.Overwrite, \"This works only with `mode` `Overwrite`\")\n ValidationUtil.isNotEmpty(\n neo4jOptions.nodeMetadata.nodeKeys,\n s\"${Neo4jOptions.NODE_KEYS} is required to define the constraints\"\n )\n }\n }\n case QueryType.RELATIONSHIP => {\n if (hasNodeOptimizations) {\n ValidationUtil.isNotEmpty(\n neo4jOptions.relationshipMetadata.source.nodeKeys,\n s\"${Neo4jOptions.RELATIONSHIP_SOURCE_NODE_KEYS} is required to define the constraints\"\n )\n ValidationUtil.isNotEmpty(\n neo4jOptions.relationshipMetadata.target.nodeKeys,\n s\"${Neo4jOptions.RELATIONSHIP_TARGET_NODE_KEYS} is required to define the constraints\"\n )\n ValidationUtil.isTrue(\n neo4jOptions.relationshipMetadata.sourceSaveMode == NodeSaveMode.Overwrite,\n s\"This works only with `${Neo4jOptions.RELATIONSHIP_SOURCE_SAVE_MODE}` `${NodeSaveMode.Overwrite}`\"\n )\n ValidationUtil.isTrue(\n neo4jOptions.relationshipMetadata.targetSaveMode == NodeSaveMode.Overwrite,\n s\"This works only with `${Neo4jOptions.RELATIONSHIP_TARGET_SAVE_MODE}` `${NodeSaveMode.Overwrite}`\"\n )\n }\n if (schemaMetadata.optimization.relConstraint != ConstraintsOptimizationType.NONE) {\n ValidationUtil.isTrue(saveMode == SaveMode.Overwrite, s\"This works only with `mode` `${SaveMode.Overwrite}`\")\n ValidationUtil.isNotEmpty(\n neo4jOptions.relationshipMetadata.relationshipKeys,\n s\"${Neo4jOptions.RELATIONSHIP_KEYS} is required to define the constraints\"\n )\n }\n }\n case _ => // do nothing\n }\n neo4jOptions.schemaMetadata.optimizationType match {\n case OptimizationType.NONE => // skip it\n case _ => neo4jOptions.query.queryType match {\n case QueryType.LABELS =>\n ValidationUtil.isTrue(saveMode == SaveMode.Overwrite, \"This works only with `mode` `SaveMode.Overwrite`\")\n case QueryType.RELATIONSHIP => {\n ValidationUtil.isTrue(\n neo4jOptions.relationshipMetadata.sourceSaveMode == NodeSaveMode.Overwrite,\n s\"This works only with `${Neo4jOptions.RELATIONSHIP_SOURCE_SAVE_MODE}` `Overwrite`\"\n )\n ValidationUtil.isTrue(\n neo4jOptions.relationshipMetadata.targetSaveMode == NodeSaveMode.Overwrite,\n s\"This works only with `${Neo4jOptions.RELATIONSHIP_TARGET_SAVE_MODE}` `Overwrite`\"\n )\n }\n }\n }\n }\n}\n\ncase class ValidateSparkMinVersion(supportedVersions: String*) extends Validation {\n\n override def validate(): Unit = {\n val sparkVersion = SparkSession.getActiveSession\n .map(_.version)\n .getOrElse(\"UNKNOWN\")\n\n ValidationUtil.isTrue(\n isSupported(sparkVersion),\n s\"\"\"Your current Spark version $sparkVersion is not supported by the current connector.\n |Please visit https://neo4j.com/developer/spark/overview/#_spark_compatibility to know which connector version you need.\n |\"\"\".stripMargin\n )\n }\n\n def isSupported(sparkVersion: String): Boolean = {\n val splittedVersion = sparkVersion.split(\"\\\\.\")\n if (sparkVersion == \"UNKNOWN\") return true\n val versions = supportedVersions\n .flatMap(_.split(\"\\\\.\").zip(splittedVersion))\n .map(t =>\n try {\n (t._1.toInt, t._2.toInt)\n } catch {\n case _: NumberFormatException => null\n }\n )\n .filter(p => p != null)\n for (t <- versions) {\n val curr = t._2\n val supported = t._1\n if (curr > supported) { // if Spark current version (step) is greater than the supported one\n return true // we can assume that the current version is a greater version, so it's supported\n }\n if (curr < supported) { // if Spark current version (step) is lower than the supported one\n return false // we can assume that the current version is a greater version, so it's supported\n }\n // if the versions are equal we can check the next step\n }\n true // this happens if the two versions are equal\n }\n}\n\ncase class ValidateConnection(neo4jOptions: Neo4jOptions, jobId: String) extends Validation {\n\n override def validate(): Unit = {\n var driverCache: DriverCache = null\n var hasError = false\n try {\n driverCache = new DriverCache(neo4jOptions.connection)\n driverCache.getOrCreate().verifyConnectivity()\n } catch {\n case e: Throwable => {\n hasError = true\n throw e\n }\n } finally {\n if (hasError) {\n Neo4jUtil.closeSafely(driverCache)\n }\n }\n }\n}\n\ncase class ValidateSaveMode(saveMode: String) extends Validation {\n\n override def validate(): Unit = {\n ValidationUtil.isTrue(\n Neo4jOptions.SUPPORTED_SAVE_MODES.contains(SaveMode.valueOf(saveMode)),\n s\"\"\"Unsupported SaveMode.\n |You provided $saveMode, supported are:\n |${Neo4jOptions.SUPPORTED_SAVE_MODES.mkString(\",\")}\n |\"\"\".stripMargin\n )\n }\n}\n\ncase class ValidateWrite(\n neo4j: Neo4j,\n neo4jOptions: Neo4jOptions,\n jobId: String,\n saveMode: SaveMode,\n customValidation: Neo4jOptions => Unit = _ => ()\n) extends Validation {\n\n override def validate(): Unit = {\n ValidationUtil.isFalse(\n neo4jOptions.session.accessMode == AccessMode.READ,\n s\"Mode READ not supported for Data Source writer\"\n )\n val cache = new DriverCache(neo4jOptions.connection)\n val schemaService = new SchemaService(neo4j, neo4jOptions, cache)\n try {\n ValidateConnection(neo4jOptions, jobId).validate()\n ValidateNeo4jOptionsConsistency(neo4jOptions).validate()\n ValidateSchemaMetadataWrite(neo4jOptions, saveMode).validate()\n\n neo4jOptions.query.queryType match {\n case QueryType.QUERY => {\n val error = schemaService.validateQuery(\n s\"\"\"WITH {} AS ${Neo4jQueryStrategy.VARIABLE_EVENT}, [] as ${Neo4jQueryStrategy.VARIABLE_SCRIPT_RESULT}\n |${neo4jOptions.query.value}\n |\"\"\".stripMargin,\n org.neo4j.driver.summary.QueryType.WRITE_ONLY,\n org.neo4j.driver.summary.QueryType.READ_WRITE\n )\n ValidationUtil.isTrue(error.isEmpty, error)\n neo4jOptions.schemaMetadata.optimizationType match {\n case OptimizationType.NONE => // are valid\n case _ => ValidationUtil.isNotValid(\n s\"\"\"With Query Type ${neo4jOptions.query.queryType} you can\n |only use `${Neo4jOptions.SCHEMA_OPTIMIZATION_TYPE}`\n |`${OptimizationType.NONE}`\n |\"\"\".stripMargin\n )\n }\n }\n case QueryType.LABELS => {\n saveMode match {\n case SaveMode.Overwrite => {\n ValidationUtil.isNotEmpty(\n neo4jOptions.nodeMetadata.nodeKeys,\n s\"${Neo4jOptions.NODE_KEYS} is required when Save Mode is Overwrite\"\n )\n }\n case _ => ()\n }\n }\n case QueryType.RELATIONSHIP => {\n ValidationUtil.isNotEmpty(\n neo4jOptions.relationshipMetadata.target.labels,\n s\"${Neo4jOptions.RELATIONSHIP_SOURCE_LABELS} is required when Save Mode is Overwrite\"\n )\n ValidationUtil.isNotEmpty(\n neo4jOptions.relationshipMetadata.target.labels,\n s\"${Neo4jOptions.RELATIONSHIP_TARGET_LABELS} is required when Save Mode is Overwrite\"\n )\n }\n }\n neo4jOptions.script.foreach(query =>\n ValidationUtil.isTrue(\n schemaService.isValidQuery(query),\n s\"The following query inside the `${Neo4jOptions.SCRIPT}` is not valid, please check the syntax: $query\"\n )\n )\n\n customValidation(neo4jOptions)\n } finally {\n schemaService.close()\n cache.close()\n }\n }\n}\n\ncase class ValidateRead(neo4j: Neo4j, neo4jOptions: Neo4jOptions, jobId: String) extends Validation {\n\n override def validate(): Unit = {\n val cache = new DriverCache(neo4jOptions.connection)\n val schemaService = new SchemaService(neo4j, neo4jOptions, cache)\n try {\n ValidateConnection(neo4jOptions, jobId).validate()\n ValidateNeo4jOptionsConsistency(neo4jOptions).validate()\n\n neo4jOptions.query.queryType match {\n case QueryType.LABELS => {\n ValidationUtil.isNotEmpty(\n neo4jOptions.nodeMetadata.labels,\n s\"You need to set the ${QueryType.LABELS.toString.toLowerCase} option\"\n )\n }\n case QueryType.RELATIONSHIP => {\n ValidationUtil.isNotBlank(\n neo4jOptions.relationshipMetadata.relationshipType,\n s\"You need to set the ${QueryType.RELATIONSHIP.toString.toLowerCase} option\"\n )\n\n ValidationUtil.isNotEmpty(\n neo4jOptions.relationshipMetadata.source.labels,\n s\"You need to set the ${Neo4jOptions.RELATIONSHIP_SOURCE_LABELS} option\"\n )\n\n ValidationUtil.isNotEmpty(\n neo4jOptions.relationshipMetadata.target.labels,\n s\"You need to set the ${Neo4jOptions.RELATIONSHIP_TARGET_LABELS} option\"\n )\n }\n case QueryType.QUERY => {\n ValidationUtil.isFalse(\n neo4jOptions.query.value.matches(\"(?si).*(LIMIT \\\\d+|SKIP ?\\\\d+)\\\\s*\\\\z\"),\n \"SKIP/LIMIT are not allowed at the end of the query\"\n )\n val queryError = schemaService.validateQuery(\n s\"\"\"WITH [] as ${Neo4jQueryStrategy.VARIABLE_SCRIPT_RESULT}\n |${neo4jOptions.query.value}\n |\"\"\".stripMargin,\n org.neo4j.driver.summary.QueryType.READ_ONLY\n )\n ValidationUtil.isTrue(queryError.isEmpty, queryError)\n if (neo4jOptions.queryMetadata.queryCount.nonEmpty) {\n if (!Neo4jUtil.isLong(neo4jOptions.queryMetadata.queryCount)) {\n val queryCountError = schemaService.validateQueryCount(neo4jOptions.queryMetadata.queryCount)\n ValidationUtil.isTrue(queryCountError.isEmpty, queryCountError)\n }\n }\n }\n case QueryType.GDS => {\n ValidationUtil.isFalse(\n neo4jOptions.query.value.contains(\".mutate\") || neo4jOptions.query.value.contains(\".write\"),\n \"You cannot execute GDS mutate or write procedure in a read query\"\n )\n ValidationUtil.isTrue(\n schemaService.isGdsProcedure(neo4jOptions.query.value),\n s\"GDS procedure ${neo4jOptions.query.value} does not exist\"\n )\n ValidationUtil.isTrue(neo4jOptions.partitions == 1, \"For GDS queries we support only one partition\")\n Validations.validate(ValidateGdsMetadata(neo4jOptions.gdsMetadata))\n }\n }\n val scriptErrors = neo4jOptions.script\n .map(schemaService.validateQuery(_))\n .filter(_.nonEmpty)\n .mkString(\"\\n\")\n ValidationUtil.isTrue(\n scriptErrors.isEmpty,\n s\"\"\"\n |The following queries inside the `${Neo4jOptions.SCRIPT}` are not valid,\n |please check their syntax:\n |$scriptErrors\n |\"\"\".stripMargin\n )\n } finally {\n schemaService.close()\n cache.close()\n }\n }\n}\n\ncase class ValidateReadNotStreaming(neo4jOptions: Neo4jOptions, jobId: String) extends Validation {\n\n override def validate(): Unit = {\n ValidationUtil.isBlank(\n neo4jOptions.streamingOptions.propertyName,\n s\"You don't need to set the `${Neo4jOptions.STREAMING_PROPERTY_NAME}` option\"\n )\n }\n}\n\n/**\n * df: this method checks for inconsistencies between provided options.\n * Ex: if we use the QueryType.LABELS, we will ignore any relationship options.\n *\n * Plus it throws an exception if no QueryType is provided.\n */\ncase class ValidateNeo4jOptionsConsistency(neo4jOptions: Neo4jOptions) extends Validation {\n\n override def validate(): Unit = {\n if (neo4jOptions.query.value.isEmpty) {\n val reqTypes = QueryType.values.map(qt => s\"`${qt.toString}`\").mkString(\", \")\n throw new IllegalArgumentException(s\"No valid option found. One of $reqTypes is required\")\n }\n\n neo4jOptions.query.queryType match {\n case QueryType.LABELS =>\n ignoreQueryMetadata(QueryType.LABELS)\n ignoreRelMetadata(QueryType.LABELS)\n ignoreGdsMetadata(QueryType.LABELS)\n case QueryType.RELATIONSHIP =>\n ignoreQueryMetadata(QueryType.RELATIONSHIP)\n ignoreNodeMetadata(QueryType.RELATIONSHIP)\n ignoreGdsMetadata(QueryType.RELATIONSHIP)\n case QueryType.QUERY => {\n ignoreNodeMetadata(QueryType.QUERY)\n ignoreRelMetadata(QueryType.QUERY)\n ignoreGdsMetadata(QueryType.QUERY)\n }\n case QueryType.GDS =>\n ignoreQueryMetadata(QueryType.GDS)\n ignoreNodeMetadata(QueryType.GDS)\n ignoreRelMetadata(QueryType.GDS)\n }\n }\n\n private def ignoreGdsMetadata(queryType: QueryType.Value): Unit = {\n if (!neo4jOptions.gdsMetadata.parameters.isEmpty) {\n ignoreOption(Neo4jOptions.QUERY_COUNT, queryType.toString.toLowerCase(Locale.ENGLISH))\n }\n }\n\n private def ignoreQueryMetadata(queryType: QueryType.Value): Unit = {\n if (neo4jOptions.queryMetadata.queryCount.nonEmpty) {\n ignoreOption(Neo4jOptions.QUERY_COUNT, queryType.toString.toLowerCase(Locale.ENGLISH))\n }\n }\n\n private def ignoreRelMetadata(queryType: QueryType.Value): Unit = {\n val optName = queryTypeAsOptionString(queryType)\n if (neo4jOptions.relationshipMetadata.source.labels.nonEmpty) {\n ignoreOption(Neo4jOptions.RELATIONSHIP_SOURCE_LABELS, optName)\n }\n if (neo4jOptions.relationshipMetadata.source.properties.nonEmpty) {\n ignoreOption(Neo4jOptions.RELATIONSHIP_SOURCE_NODE_PROPS, optName)\n }\n if (neo4jOptions.relationshipMetadata.source.nodeKeys.nonEmpty) {\n ignoreOption(Neo4jOptions.RELATIONSHIP_SOURCE_NODE_KEYS, optName)\n }\n if (neo4jOptions.relationshipMetadata.target.labels.nonEmpty) {\n ignoreOption(Neo4jOptions.RELATIONSHIP_TARGET_LABELS, optName)\n }\n if (neo4jOptions.relationshipMetadata.target.properties.nonEmpty) {\n ignoreOption(Neo4jOptions.RELATIONSHIP_TARGET_NODE_PROPS, optName)\n }\n if (neo4jOptions.relationshipMetadata.target.nodeKeys.nonEmpty) {\n ignoreOption(Neo4jOptions.RELATIONSHIP_TARGET_NODE_KEYS, optName)\n }\n }\n\n private def queryTypeAsOptionString(queryType: util.QueryType.Value): String =\n queryType.toString.toLowerCase(Locale.ENGLISH)\n\n private def ignoreNodeMetadata(queryType: QueryType.Value): Unit = {\n val optName = queryTypeAsOptionString(queryType)\n if (neo4jOptions.nodeMetadata.nodeKeys.nonEmpty) {\n ignoreOption(Neo4jOptions.NODE_KEYS, optName)\n }\n if (neo4jOptions.nodeMetadata.properties.nonEmpty) {\n ignoreOption(Neo4jOptions.NODE_PROPS, optName)\n }\n }\n}\n\ncase class ValidateGdsMetadata(neo4jGdsMetadata: Neo4jGdsMetadata) extends Validation {\n\n override def validate(): Unit = {\n val hasGraphName = neo4jGdsMetadata.parameters.get(\"graphName\") != null\n val hasGraphNameOrConfiguration = neo4jGdsMetadata.parameters.get(\"graphNameOrConfiguration\") != null\n ValidationUtil.isTrue(\n hasGraphName || hasGraphNameOrConfiguration,\n \"One between gds.graphName or gds.graphNameOrConfiguration is required\"\n )\n }\n}\n\ncase class ValidateReadStreaming(neo4j: Neo4j, neo4jOptions: Neo4jOptions, jobId: String) extends Validation {\n\n override def validate(): Unit = {\n val cache = new DriverCache(neo4jOptions.connection)\n val schemaService = new SchemaService(neo4j, neo4jOptions, cache)\n try {\n ValidationUtil.isTrue(\n neo4jOptions.partitions == 1,\n \"For Spark Structured Streaming we support only one partition\"\n )\n neo4jOptions.query.queryType match {\n case QueryType.QUERY => {\n ValidationUtil.isTrue(\n schemaService.isValidQuery(neo4jOptions.streamingOptions.queryOffset, summary.QueryType.READ_ONLY),\n \"\"\"\n |Please set `streaming.query.offset` with a valid Cypher READ_ONLY query\n |that returns a long value i.e.\n |MATCH (p:MyLabel)\n |RETURN max(p.timestamp)\n |\"\"\".stripMargin\n )\n }\n case _ =>\n }\n } finally {\n schemaService.close()\n cache.close()\n }\n }\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/writer/BaseDataWriter.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.writer\n\nimport org.apache.spark.internal.Logging\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.connector.metric.CustomTaskMetric\nimport org.apache.spark.sql.connector.write.DataWriter\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.driver.Session\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.Values\nimport org.neo4j.driver.exceptions.ServiceUnavailableException\nimport org.neo4j.spark.service._\nimport org.neo4j.spark.util.DriverCache\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.Neo4jUtil.closeSafely\nimport org.neo4j.spark.util.Neo4jUtil.isRetryableException\n\nimport java.io.Closeable\nimport java.time.Duration\nimport java.util\nimport java.util.concurrent.CountDownLatch\nimport java.util.concurrent.locks.LockSupport\n\nimport scala.annotation.tailrec\nimport scala.collection.JavaConverters._\n\nabstract class BaseDataWriter(\n neo4j: Neo4j,\n jobId: String,\n partitionId: Int,\n structType: StructType,\n saveMode: SaveMode,\n options: Neo4jOptions,\n scriptResult: java.util.List[java.util.Map[String, AnyRef]]\n) extends Logging with Closeable with DataWriter[InternalRow] {\n\n private val STOPPED_THREAD_EXCEPTION_MESSAGE =\n \"Connection to the database terminated. Thread interrupted while committing the transaction\"\n\n private val driverCache: DriverCache = new DriverCache(options.connection)\n\n private var transaction: Transaction = _\n private var session: Session = _\n\n private val mappingService = new MappingService(new Neo4jWriteMappingStrategy(options), options)\n\n private val batch: util.List[java.util.Map[String, Object]] = new util.ArrayList[util.Map[String, Object]]()\n\n private val retries = new CountDownLatch(options.transactionSettings.retries)\n\n private val query: String =\n new Neo4jQueryService(options, new Neo4jQueryWriteStrategy(neo4j, saveMode)).createQuery()\n\n private val metrics = DataWriterMetrics()\n\n private var skipped = 0\n\n def write(record: InternalRow): Unit = {\n val mapped = mappingService.convert(record, structType)\n mapped match {\n case Some(m) => batch.add(m)\n case None =>\n skipped += 1\n }\n if (batch.size() == options.transactionSettings.batchSize) {\n writeBatch()\n }\n }\n\n @tailrec\n private def writeBatch(): Unit = {\n try {\n if (session == null || !session.isOpen) {\n session = driverCache.getOrCreate().session(options.session.toNeo4jSession())\n }\n if (transaction == null || !transaction.isOpen) {\n transaction = session.beginTransaction(options.toNeo4jTransactionConfig)\n }\n log.info(\n s\"\"\"Writing a batch of ${batch.size()} elements to Neo4j,\n |for jobId=$jobId and partitionId=$partitionId\n |with query: $query\n |\"\"\".stripMargin\n )\n val result = transaction.run(\n query,\n Values.value(Map[String, AnyRef](\n Neo4jQueryStrategy.VARIABLE_EVENTS -> batch,\n Neo4jQueryStrategy.VARIABLE_SCRIPT_RESULT -> scriptResult\n ).asJava)\n )\n val summary = result.consume()\n val counters = summary.counters()\n if (log.isDebugEnabled) {\n log.debug(\n s\"\"\"Batch saved into Neo4j data with:\n | - nodes created: ${counters.nodesCreated()}\n | - nodes deleted: ${counters.nodesDeleted()}\n | - relationships created: ${counters.relationshipsCreated()}\n | - relationships deleted: ${counters.relationshipsDeleted()}\n | - properties set: ${counters.propertiesSet()}\n | - labels added: ${counters.labelsAdded()}\n | - labels removed: ${counters.labelsRemoved()}\n |\"\"\".stripMargin\n )\n }\n transaction.commit()\n\n if (skipped > 0) {\n log.info(s\"Skipped $skipped rows that contained null values in one of their key property values.\")\n skipped = 0\n }\n\n // update metrics\n metrics.applyCounters(batch.size(), counters)\n\n closeSafely(transaction)\n batch.clear()\n } catch {\n case e: Throwable =>\n if (options.transactionSettings.shouldFailOn(e)) {\n log.error(\"unable to write batch due to explicitly configured failure condition\", e)\n throw e\n }\n\n if (isRetryableException(e) && retries.getCount > 0) {\n retries.countDown()\n log.info(\n s\"encountered a transient exception while writing batch, retrying ${options.transactionSettings.retries - retries.getCount} time\",\n e\n )\n close()\n LockSupport.parkNanos(Duration.ofMillis(options.transactionSettings.retryTimeout).toNanos)\n writeBatch()\n } else {\n logAndThrowException(e)\n }\n }\n }\n\n /**\n * df: we check if the thrown exception is STOPPED_THREAD_EXCEPTION. This is the\n * exception that is thrown when the streaming query is interrupted, we don't want to cause\n * any error in this case. The transaction are rolled back automatically.\n */\n private def logAndThrowException(e: Throwable): Unit = {\n if (e.isInstanceOf[ServiceUnavailableException] && e.getMessage == STOPPED_THREAD_EXCEPTION_MESSAGE) {\n logWarning(e.getMessage)\n } else {\n logError(\"unable to write batch\", e)\n }\n\n throw e\n }\n\n def commit(): Null = {\n writeBatch()\n close()\n null\n }\n\n def abort(): Unit = {\n if (transaction != null && transaction.isOpen) {\n try {\n transaction.rollback()\n } catch {\n case e: Throwable => log.warn(\"Cannot rollback the transaction because of the following exception\", e)\n }\n }\n close()\n }\n\n def close(): Unit = {\n closeSafely(transaction, log)\n closeSafely(session, log)\n }\n\n override def currentMetricsValues(): Array[CustomTaskMetric] = metrics.metricValues()\n}\n" }, { "path": "common/src/main/scala/org/neo4j/spark/writer/DataWriterMetrics.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.writer\n\nimport org.apache.spark.sql.connector.metric.CustomMetric\nimport org.apache.spark.sql.connector.metric.CustomSumMetric\nimport org.apache.spark.sql.connector.metric.CustomTaskMetric\nimport org.neo4j.driver.summary.SummaryCounters\nimport org.neo4j.spark.writer.DataWriterMetrics.LABELS_ADDED\nimport org.neo4j.spark.writer.DataWriterMetrics.LABELS_REMOVED\nimport org.neo4j.spark.writer.DataWriterMetrics.NODES_CREATED\nimport org.neo4j.spark.writer.DataWriterMetrics.NODES_DELETED\nimport org.neo4j.spark.writer.DataWriterMetrics.PROPERTIES_SET\nimport org.neo4j.spark.writer.DataWriterMetrics.RECORDS_WRITTEN\nimport org.neo4j.spark.writer.DataWriterMetrics.RELATIONSHIPS_CREATED\nimport org.neo4j.spark.writer.DataWriterMetrics.RELATIONSHIPS_DELETED\n\nimport java.util.concurrent.atomic.AtomicLong\n\ncase class DataWriterMetric(name: String, value: Long) extends CustomTaskMetric {}\n\nclass DataWriterMetrics private (\n recordsProcessed: AtomicLong,\n nodesCreated: AtomicLong,\n nodesDeleted: AtomicLong,\n relationshipsCreated: AtomicLong,\n relationshipsDeleted: AtomicLong,\n propertiesSet: AtomicLong,\n labelsAdded: AtomicLong,\n labelsRemoved: AtomicLong\n) {\n\n def applyCounters(recordsWritten: Long, counters: SummaryCounters): Unit = {\n this.recordsProcessed.addAndGet(recordsWritten)\n this.nodesCreated.addAndGet(counters.nodesCreated())\n this.nodesDeleted.addAndGet(counters.nodesDeleted())\n this.relationshipsCreated.addAndGet(counters.relationshipsCreated())\n this.relationshipsDeleted.addAndGet(counters.relationshipsDeleted())\n this.propertiesSet.addAndGet(counters.propertiesSet())\n this.labelsAdded.addAndGet(counters.labelsAdded())\n this.labelsRemoved.addAndGet(counters.labelsRemoved())\n }\n\n def metricValues(): Array[CustomTaskMetric] = {\n List[CustomTaskMetric](\n DataWriterMetric(RECORDS_WRITTEN, recordsProcessed.longValue()),\n DataWriterMetric(NODES_CREATED, nodesCreated.longValue()),\n DataWriterMetric(NODES_DELETED, nodesDeleted.longValue()),\n DataWriterMetric(RELATIONSHIPS_CREATED, relationshipsCreated.longValue()),\n DataWriterMetric(RELATIONSHIPS_DELETED, relationshipsDeleted.longValue()),\n DataWriterMetric(PROPERTIES_SET, propertiesSet.longValue()),\n DataWriterMetric(LABELS_ADDED, labelsAdded.longValue()),\n DataWriterMetric(LABELS_REMOVED, labelsRemoved.longValue())\n ).toArray\n }\n\n}\n\nobject DataWriterMetrics {\n final val RECORDS_WRITTEN = \"neo4jMetrics.recordsWritten\"\n final val RECORDS_WRITTEN_DESCRIPTION = \"number of records written\"\n final val NODES_CREATED = \"neo4jMetrics.nodesCreated\"\n final val NODES_CREATED_DESCRIPTION = \"number of nodes created\"\n final val NODES_DELETED = \"neo4jMetrics.nodesDeleted\"\n final val NODES_DELETED_DESCRIPTION = \"number of nodes deleted\"\n final val RELATIONSHIPS_CREATED = \"neo4jMetrics.relationshipsCreated\"\n final val RELATIONSHIPS_CREATED_DESCRIPTION = \"number of relationships created\"\n final val RELATIONSHIPS_DELETED = \"neo4jMetrics.relationshipsDeleted\"\n final val RELATIONSHIPS_DELETED_DESCRIPTION = \"number of relationships deleted\"\n final val PROPERTIES_SET = \"neo4jMetrics.propertiesSet\"\n final val PROPERTIES_SET_DESCRIPTION = \"number of properties set\"\n final val LABELS_ADDED = \"neo4jMetrics.labelsAdded\"\n final val LABELS_ADDED_DESCRIPTION = \"number of labels added\"\n final val LABELS_REMOVED = \"neo4jMetrics.labelsRemoved\"\n final val LABELS_REMOVED_DESCRIPTION = \"number of labels removed\"\n\n def apply(): DataWriterMetrics = {\n new DataWriterMetrics(\n new AtomicLong(0),\n new AtomicLong(0),\n new AtomicLong(0),\n new AtomicLong(0),\n new AtomicLong(0),\n new AtomicLong(0),\n new AtomicLong(0),\n new AtomicLong(0)\n )\n }\n\n def metricDeclarations(): Array[CustomMetric] = {\n List[CustomMetric](\n new RecordsWrittenMetric,\n new NodesCreatedMetric,\n new NodesDeletedMetric,\n new RelationshipsCreatedMetric,\n new RelationshipsDeletedMetric,\n new PropertiesSetMetric,\n new LabelsAddedMetric,\n new LabelsRemovedMetric\n ).toArray\n }\n\n}\n\nclass RecordsWrittenMetric extends CustomSumMetric {\n override def name(): String = DataWriterMetrics.RECORDS_WRITTEN\n\n override def description(): String = DataWriterMetrics.RECORDS_WRITTEN_DESCRIPTION\n}\n\nclass NodesCreatedMetric extends CustomSumMetric {\n override def name(): String = DataWriterMetrics.NODES_CREATED\n\n override def description(): String = DataWriterMetrics.NODES_CREATED_DESCRIPTION\n}\n\nclass NodesDeletedMetric extends CustomSumMetric {\n override def name(): String = DataWriterMetrics.NODES_DELETED\n\n override def description(): String = DataWriterMetrics.NODES_DELETED_DESCRIPTION\n}\n\nclass RelationshipsCreatedMetric extends CustomSumMetric {\n override def name(): String = DataWriterMetrics.RELATIONSHIPS_CREATED\n\n override def description(): String = DataWriterMetrics.RELATIONSHIPS_CREATED_DESCRIPTION\n}\n\nclass RelationshipsDeletedMetric extends CustomSumMetric {\n override def name(): String = DataWriterMetrics.RELATIONSHIPS_DELETED\n\n override def description(): String = DataWriterMetrics.RELATIONSHIPS_DELETED_DESCRIPTION\n}\n\nclass PropertiesSetMetric extends CustomSumMetric {\n override def name(): String = DataWriterMetrics.PROPERTIES_SET\n\n override def description(): String = DataWriterMetrics.PROPERTIES_SET_DESCRIPTION\n}\n\nclass LabelsAddedMetric extends CustomSumMetric {\n override def name(): String = DataWriterMetrics.LABELS_ADDED\n\n override def description(): String = DataWriterMetrics.LABELS_ADDED_DESCRIPTION\n}\n\nclass LabelsRemovedMetric extends CustomSumMetric {\n override def name(): String = DataWriterMetrics.LABELS_REMOVED\n\n override def description(): String = DataWriterMetrics.LABELS_REMOVED_DESCRIPTION\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/CommonTestSuiteIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.junit.runner.RunWith\nimport org.junit.runners.Suite\nimport org.neo4j.spark.service.SchemaServiceTSE\n\n@RunWith(classOf[Suite])\n@Suite.SuiteClasses(Array(\n classOf[SchemaServiceTSE]\n))\nclass CommonTestSuiteIT extends SparkConnectorScalaSuiteIT {}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/CommonTestSuiteWithApocIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.junit.runner.RunWith\nimport org.junit.runners.Suite\nimport org.neo4j.spark.service.SchemaServiceWithApocTSE\n\n@RunWith(classOf[Suite])\n@Suite.SuiteClasses(Array(\n classOf[SchemaServiceWithApocTSE]\n))\nclass CommonTestSuiteWithApocIT extends SparkConnectorScalaSuiteWithApocIT {}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/service/AuthenticationTest.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.service\n\nimport org.junit.Test\nimport org.junit.runner.RunWith\nimport org.mockito.ArgumentMatchers\nimport org.mockito.ArgumentMatchers._\nimport org.mockito.Mockito.times\nimport org.neo4j.driver.AuthTokens\nimport org.neo4j.driver.Config\nimport org.neo4j.driver.GraphDatabase\nimport org.neo4j.spark.util.DriverCache\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.powermock.api.mockito.PowerMockito\nimport org.powermock.core.classloader.annotations.PowerMockIgnore\nimport org.powermock.core.classloader.annotations.PrepareForTest\nimport org.powermock.modules.junit4.PowerMockRunner\nimport org.testcontainers.shaded.com.google.common.io.BaseEncoding\n\nimport java.net.URI\nimport java.util\n\n@PrepareForTest(Array(classOf[GraphDatabase]))\n@RunWith(classOf[PowerMockRunner])\n@PowerMockIgnore(value = Array(\"javax.management.*\"))\nclass AuthenticationTest {\n\n @Test\n def testLdapConnectionToken(): Unit = {\n val token = BaseEncoding.base64.encode(\"user:password\".getBytes)\n val options = new util.HashMap[String, String]\n options.put(\"url\", \"bolt://localhost:7687\")\n options.put(\"authentication.type\", \"custom\")\n options.put(\"authentication.custom.credentials\", token)\n options.put(\"labels\", \"Person\")\n\n val neo4jOptions = new Neo4jOptions(options)\n val neo4jDriverOptions = neo4jOptions.connection\n val driverCache = new DriverCache(neo4jDriverOptions)\n\n PowerMockito.spy(classOf[GraphDatabase])\n\n driverCache.getOrCreate()\n\n PowerMockito.verifyStatic(classOf[GraphDatabase], times(1))\n GraphDatabase.driver(any[URI](), ArgumentMatchers.eq(AuthTokens.custom(\"\", token, \"\", \"\")), any(classOf[Config]))\n }\n\n @Test\n def testBearerAuthToken(): Unit = {\n val token = BaseEncoding.base64.encode(\"user:password\".getBytes)\n val options = new util.HashMap[String, String]\n options.put(\"url\", \"bolt://localhost:7687\")\n options.put(\"authentication.type\", \"bearer\")\n options.put(\"authentication.bearer.token\", token)\n\n val neo4jOptions = new Neo4jOptions(options)\n val neo4jDriverOptions = neo4jOptions.connection\n val driverCache = new DriverCache(neo4jDriverOptions)\n\n PowerMockito.spy(classOf[GraphDatabase])\n\n driverCache.getOrCreate()\n\n PowerMockito.verifyStatic(classOf[GraphDatabase], times(1))\n GraphDatabase.driver(any[URI](), ArgumentMatchers.eq(AuthTokens.bearer(token)), any())\n }\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/service/Neo4jQueryServiceIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.service\n\nimport org.apache.spark.sql.connector.expressions.aggregate.Count\nimport org.apache.spark.sql.connector.expressions.aggregate.Max\nimport org.apache.spark.sql.connector.expressions.aggregate.Min\nimport org.apache.spark.sql.connector.expressions.aggregate.Sum\nimport org.junit.After\nimport org.junit.FixMethodOrder\nimport org.junit.Test\nimport org.junit.runners.MethodSorters\nimport org.neo4j.spark.SparkConnectorScalaSuiteWithGdsBase\nimport org.neo4j.spark.SparkConnectorScalaSuiteWithGdsBase.neo4j\nimport org.neo4j.spark.util.DriverCache\nimport org.neo4j.spark.util.DummyNamedReference\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.scalatest.matchers.must.Matchers.convertToAnyMustWrapper\nimport org.scalatest.matchers.must.Matchers.endWith\n\nimport scala.language.postfixOps\n\n@FixMethodOrder(MethodSorters.JVM)\nclass Neo4jQueryServiceIT extends SparkConnectorScalaSuiteWithGdsBase {\n\n @After\n def cleanUp(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n new DriverCache(neo4jOptions.connection).close()\n }\n\n @Test\n def testShouldDoAggregationOnGDS(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n options.put(\"gds\", \"gds.pageRank.stream\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val field = new DummyNamedReference(\"score\")\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty,\n PartitionPagination.EMPTY,\n List(\n \"nodeId\",\n \"MAX(score)\",\n \"MIN(score)\",\n \"COUNT(score)\",\n \"COUNT(DISTINCT score)\",\n \"SUM(score)\",\n \"SUM(DISTINCT score)\"\n ),\n Array(\n new Max(field),\n new Min(field),\n new Sum(field, false),\n new Count(field, false),\n new Count(field, true),\n new Sum(field, false),\n new Sum(field, true)\n )\n )\n ).createQuery()\n\n query must endWith(\n \"\"\"CALL gds.pageRank.stream($graphName)\n |YIELD nodeId, score\n |RETURN nodeId AS nodeId, max(score) AS `MAX(score)`, min(score) AS `MIN(score)`, count(score) AS `COUNT(score)`, count(DISTINCT score) AS `COUNT(DISTINCT score)`, sum(score) AS `SUM(score)`, sum(DISTINCT score) AS `SUM(DISTINCT score)`\"\"\"\n .stripMargin\n .replaceAll(\"\\n\", \" \")\n )\n }\n\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/service/Neo4jQueryServiceTest.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.service\n\nimport junitparams.JUnitParamsRunner\nimport junitparams.Parameters\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.connector.expressions.Expression\nimport org.apache.spark.sql.connector.expressions.NullOrdering\nimport org.apache.spark.sql.connector.expressions.SortDirection\nimport org.apache.spark.sql.connector.expressions.SortOrder\nimport org.apache.spark.sql.connector.expressions.aggregate.Count\nimport org.apache.spark.sql.connector.expressions.aggregate.Max\nimport org.apache.spark.sql.connector.expressions.aggregate.Min\nimport org.apache.spark.sql.connector.expressions.aggregate.Sum\nimport org.apache.spark.sql.sources._\nimport org.junit.Assert._\nimport org.junit.Test\nimport org.junit.runner.RunWith\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.caniuse.Neo4jDeploymentType.SELF_MANAGED\nimport org.neo4j.caniuse.Neo4jEdition\nimport org.neo4j.caniuse.Neo4jEdition.COMMUNITY\nimport org.neo4j.caniuse.Neo4jEdition.ENTERPRISE\nimport org.neo4j.caniuse.Neo4jVersion\nimport org.neo4j.spark.config.TopN\nimport org.neo4j.spark.util.DummyNamedReference\nimport org.neo4j.spark.util.Neo4jImplicits.CypherImplicits\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.QueryType\n\nimport scala.collection.immutable.HashMap\n\n@RunWith(classOf[JUnitParamsRunner])\nclass Neo4jQueryServiceTest {\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testNodeOneLabel(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j)).createQuery()\n\n assertEquals(s\"${prefix}MATCH (n:`Person`) RETURN n\", query)\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testNodeMultipleLabels(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \":Person:Player:Midfield\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j)).createQuery()\n\n assertEquals(s\"${prefix}MATCH (n:`Person`:`Player`:`Midfield`) RETURN n\", query)\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testNodeMultipleLabelsWithPartitions(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \":Person:Player:Midfield\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n partitionPagination = PartitionPagination(0, 0, TopN(100))\n )\n ).createQuery()\n\n assertEquals(s\"${prefix}MATCH (n:`Person`:`Player`:`Midfield`) RETURN n LIMIT 100\", query)\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testNodeOneLabelWithOneSelectedColumn(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(neo4j, Array.empty[Filter], PartitionPagination.EMPTY, Seq(\"name\"))\n ).createQuery()\n\n assertEquals(s\"${prefix}MATCH (n:`Person`) RETURN n.name AS name\", query)\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testNodeOneLabelWithMultipleColumnSelected(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(neo4j, Array.empty[Filter], PartitionPagination.EMPTY, List(\"name\", \"bornDate\"))\n ).createQuery()\n\n assertEquals(s\"${prefix}MATCH (n:`Person`) RETURN n.name AS name, n.bornDate AS bornDate\", query)\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testNodeOneLabelWithInternalIdSelected(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(neo4j, Array.empty[Filter], PartitionPagination.EMPTY, List(\"\"))\n ).createQuery()\n\n assertEquals(s\"${prefix}MATCH (n:`Person`) RETURN id(n) AS ``\", query)\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testNodeFilterEqualTo(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val filters: Array[Filter] = Array[Filter](\n EqualTo(\"name\", \"John Doe\")\n )\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j, filters)).createQuery()\n\n val paramName = \"$\" + \"name\".toParameterName(\"John Doe\")\n\n assertEquals(s\"${prefix}MATCH (n:`Person`) WHERE n.name = $paramName RETURN n\", query)\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testNodeFilterEqualNullSafe(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val filters: Array[Filter] = Array[Filter](\n EqualNullSafe(\"name\", \"John Doe\"),\n EqualTo(\"age\", 36)\n )\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j, filters)).createQuery()\n\n val nameParameterName = \"$\" + \"name\".toParameterName(\"John Doe\")\n val ageParameterName = \"$\" + \"age\".toParameterName(36)\n\n assertEquals(\n s\"\"\"${prefix}MATCH (n:`Person`)\n | WHERE (((n.name IS NULL AND $nameParameterName IS NULL)\n | OR n.name = $nameParameterName) AND n.age = $ageParameterName)\n | RETURN n\"\"\".stripMargin.replaceAll(\"\\n\", \"\"),\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testNodeFilterEqualNullSafeWithNullValue(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val filters: Array[Filter] = Array[Filter](\n EqualNullSafe(\"name\", null),\n EqualTo(\"age\", 36)\n )\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j, filters)).createQuery()\n\n val nameParameterName = \"$\" + \"name\".toParameterName(null)\n val ageParameterName = \"$\" + \"age\".toParameterName(36)\n\n assertEquals(\n s\"${prefix}MATCH (n:`Person`) WHERE (((n.name IS NULL AND $nameParameterName IS NULL) OR n.name = $nameParameterName) AND n.age = $ageParameterName) RETURN n\",\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testNodeFilterStartsEndsWith(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val filters: Array[Filter] = Array[Filter](\n StringStartsWith(\"name\", \"Person Name\"),\n StringEndsWith(\"name\", \"Person Surname\")\n )\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j, filters)).createQuery()\n\n val nameOneParameterName = \"$\" + \"name\".toParameterName(\"Person Name\")\n val nameTwoParameterName = \"$\" + \"name\".toParameterName(\"Person Surname\")\n\n assertEquals(\n s\"\"\"${prefix}MATCH (n:`Person`)\n | WHERE (n.name STARTS WITH $nameOneParameterName\n | AND n.name ENDS WITH $nameTwoParameterName)\n | RETURN n\"\"\".stripMargin.replaceAll(\"\\n\", \"\"),\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testRelationshipWithOneColumnSelected(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"KNOWS\")\n options.put(\"relationship.nodes.map\", \"false\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty[Filter],\n PartitionPagination.EMPTY,\n List(\"source.name\")\n )\n ).createQuery()\n\n assertEquals(\n s\"${prefix}MATCH (source:`Person`) \" +\n \"MATCH (target:`Person`) \" +\n \"MATCH (source)-[rel:`KNOWS`]->(target) RETURN source.name AS `source.name`\",\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testRelationshipWithMoreColumnSelected(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"KNOWS\")\n options.put(\"relationship.nodes.map\", \"false\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty[Filter],\n PartitionPagination.EMPTY,\n List(\"source.name\", \"\")\n )\n ).createQuery()\n\n assertEquals(\n s\"${prefix}MATCH (source:`Person`) \" +\n \"MATCH (target:`Person`) \" +\n \"MATCH (source)-[rel:`KNOWS`]->(target) RETURN source.name AS `source.name`, id(source) AS ``\",\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testRelationshipWithMoreColumnSelectedWithPartitions(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"KNOWS\")\n options.put(\"relationship.nodes.map\", \"false\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty[Filter],\n PartitionPagination(0, 0, TopN(limit = 100)),\n List(\"source.name\", \"\")\n )\n ).createQuery()\n\n assertEquals(\n s\"\"\"${prefix}MATCH (source:`Person`)\n |MATCH (target:`Person`)\n |MATCH (source)-[rel:`KNOWS`]->(target)\n |RETURN source.name AS `source.name`, id(source) AS ``\n |LIMIT 100\"\"\"\n .stripMargin\n .replace(System.lineSeparator(), \" \"),\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testRelationshipWithMoreColumnsSelected(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"KNOWS\")\n options.put(\"relationship.nodes.map\", \"false\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty[Filter],\n PartitionPagination.EMPTY,\n List(\"source.name\", \"source.id\", \"rel.someprops\", \"target.date\")\n )\n ).createQuery()\n\n assertEquals(\n s\"${prefix}MATCH (source:`Person`) \" +\n \"MATCH (target:`Person`) \" +\n \"MATCH (source)-[rel:`KNOWS`]->(target) RETURN source.name AS `source.name`, source.id AS `source.id`, rel.someprops AS `rel.someprops`, target.date AS `target.date`\",\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testRelationshipFilterEqualTo(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"KNOWS\")\n options.put(\"relationship.nodes.map\", \"false\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val filters: Array[Filter] = Array[Filter](\n EqualTo(\"source.name\", \"John Doe\")\n )\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j, filters)).createQuery()\n\n val parameterName = \"$\" + \"source.name\".toParameterName(\"John Doe\")\n\n assertEquals(\n s\"${prefix}MATCH (source:`Person`) \" +\n \"MATCH (target:`Person`) \" +\n s\"MATCH (source)-[rel:`KNOWS`]->(target) WHERE source.name = $parameterName RETURN rel, source AS source, target AS target\",\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testRelationshipFilterNotEqualTo(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"KNOWS\")\n options.put(\"relationship.nodes.map\", \"false\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val filters: Array[Filter] = Array[Filter](\n Or(EqualTo(\"source.name\", \"John Doe\"), EqualTo(\"target.name\", \"John Doe\"))\n )\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j, filters)).createQuery()\n\n val paramOneName = \"$\" + \"source.name\".toParameterName(\"John Doe\")\n val paramTwoName = \"$\" + \"target.name\".toParameterName(\"John Doe\")\n\n assertEquals(\n s\"${prefix}MATCH (source:`Person`) \" +\n \"MATCH (target:`Person`) \" +\n s\"MATCH (source)-[rel:`KNOWS`]->(target) WHERE (source.name = $paramOneName OR target.name = $paramTwoName) RETURN rel, source AS source, target AS target\",\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testRelationshipAndFilterEqualTo(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"KNOWS\")\n options.put(\"relationship.nodes.map\", \"true\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val filters: Array[Filter] = Array[Filter](\n EqualTo(\"source.id\", \"14\"),\n EqualTo(\"target.id\", \"16\")\n )\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j, filters)).createQuery()\n\n val sourceIdParameterName = \"$\" + \"source.id\".toParameterName(14)\n val targetIdParameterName = \"$\" + \"target.id\".toParameterName(16)\n\n assertEquals(\n s\"\"\"${prefix}MATCH (source:`Person`)\n | MATCH (target:`Person`)\n | MATCH (source)-[rel:`KNOWS`]->(target)\n | WHERE (source.id = $sourceIdParameterName AND target.id = $targetIdParameterName)\n | RETURN rel, source AS source, target AS target\n |\"\"\".stripMargin.replaceAll(\"\\n\", \"\"),\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testComplexNodeConditions(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"labels\", \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val filters: Array[Filter] = Array[Filter](\n Or(EqualTo(\"name\", \"John Doe\"), EqualTo(\"name\", \"John Scofield\")),\n Or(EqualTo(\"age\", 15), GreaterThanOrEqual(\"age\", 18)),\n Or(Not(EqualTo(\"age\", 22)), Not(LessThan(\"age\", 11)))\n )\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j, filters)).createQuery()\n\n val parameterNames: Map[String, String] = HashMap(\n \"name_1\" -> \"$\".concat(\"name\".toParameterName(\"John Doe\")),\n \"name_2\" -> \"$\".concat(\"name\".toParameterName(\"John Scofield\")),\n \"age_1\" -> \"$\".concat(\"age\".toParameterName(15)),\n \"age_2\" -> \"$\".concat(\"age\".toParameterName(18)),\n \"age_3\" -> \"$\".concat(\"age\".toParameterName(22)),\n \"age_4\" -> \"$\".concat(\"age\".toParameterName(11))\n )\n\n assertEquals(\n s\"\"\"${prefix}MATCH (n:`Person`)\n | WHERE (((n.name = ${parameterNames(\"name_1\")} OR n.name = ${parameterNames(\"name_2\")})\n | AND (n.age = ${parameterNames(\"age_1\")} OR n.age >= ${parameterNames(\"age_2\")}))\n | AND (NOT (n.age = ${parameterNames(\"age_3\")}) OR NOT (n.age < ${parameterNames(\"age_4\")})))\n | RETURN n\"\"\".stripMargin.replaceAll(\"\\n\", \"\"),\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testRelationshipFilterComplexConditionsNoMap(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"KNOWS\")\n options.put(\"relationship.nodes.map\", \"false\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Person:Customer\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val filters: Array[Filter] = Array[Filter](\n Or(\n Or(EqualTo(\"source.name\", \"John Doe\"), EqualTo(\"target.name\", \"John Doraemon\")),\n EqualTo(\"source.name\", \"Jane Doe\")\n ),\n Or(EqualTo(\"target.age\", 34), EqualTo(\"target.age\", 18)),\n EqualTo(\"rel.score\", 12)\n )\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j, filters)).createQuery()\n\n val parameterNames = Map(\n \"source.name_1\" -> \"$\".concat(\"source.name\".toParameterName(\"John Doe\")),\n \"target.name_1\" -> \"$\".concat(\"target.name\".toParameterName(\"John Doraemon\")),\n \"source.name_2\" -> \"$\".concat(\"source.name\".toParameterName(\"Jane Doe\")),\n \"target.age_1\" -> \"$\".concat(\"target.age\".toParameterName(34)),\n \"target.age_2\" -> \"$\".concat(\"target.age\".toParameterName(18)),\n \"rel.score\" -> \"$\".concat(\"rel.score\".toParameterName(12))\n )\n\n assertEquals(\n s\"\"\"${prefix}MATCH (source:`Person`)\n | MATCH (target:`Person`:`Customer`)\n | MATCH (source)-[rel:`KNOWS`]->(target)\n | WHERE ((source.name = ${parameterNames(\"source.name_1\")} OR target.name = ${\n parameterNames(\n \"target.name_1\"\n )\n } OR source.name = ${parameterNames(\"source.name_2\")})\n | AND (target.age = ${parameterNames(\"target.age_1\")} OR target.age = ${parameterNames(\"target.age_2\")})\n | AND rel.score = ${parameterNames(\"rel.score\")})\n | RETURN rel, source AS source, target AS target\"\"\".stripMargin.replaceAll(\"\\n\", \"\"),\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testRelationshipFilterComplexConditionsWithMap(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"KNOWS\")\n options.put(\"relationship.nodes.map\", \"true\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Person:Customer\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val filters: Array[Filter] = Array[Filter](\n Or(\n Or(EqualTo(\"source.name\", \"John Doe\"), EqualTo(\"target.name\", \"John Doraemon\")),\n EqualTo(\"source.name\", \"Jane Doe\")\n ),\n Or(EqualTo(\"target.age\", 34), EqualTo(\"target.age\", 18)),\n EqualTo(\"rel.score\", 12)\n )\n\n val query: String = new Neo4jQueryService(neo4jOptions, new Neo4jQueryReadStrategy(neo4j, filters)).createQuery()\n\n val parameterNames = Map(\n \"source.name_1\" -> \"$\".concat(\"source.name\".toParameterName(\"John Doe\")),\n \"target.name_1\" -> \"$\".concat(\"target.name\".toParameterName(\"John Doraemon\")),\n \"source.name_2\" -> \"$\".concat(\"source.name\".toParameterName(\"Jane Doe\")),\n \"target.age_1\" -> \"$\".concat(\"target.age\".toParameterName(34)),\n \"target.age_2\" -> \"$\".concat(\"target.age\".toParameterName(18)),\n \"rel.score\" -> \"$\".concat(\"rel.score\".toParameterName(12))\n )\n\n assertEquals(\n s\"\"\"${prefix}MATCH (source:`Person`)\n | MATCH (target:`Person`:`Customer`)\n | MATCH (source)-[rel:`KNOWS`]->(target)\n | WHERE ((source.name = ${parameterNames(\"source.name_1\")} OR target.name = ${\n parameterNames(\n \"target.name_1\"\n )\n } OR source.name = ${parameterNames(\"source.name_2\")})\n | AND (target.age = ${parameterNames(\"target.age_1\")} OR target.age = ${parameterNames(\"target.age_2\")})\n | AND rel.score = ${parameterNames(\"rel.score\")})\n | RETURN rel, source AS source, target AS target\n |\"\"\".stripMargin.replaceAll(\"\\n\", \"\"),\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testCompoundKeysForNodes(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"labels\", \"Location\")\n options.put(\"node.keys\", \"LocationName:name,LocationType:type,FeatureID:featureId\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String =\n new Neo4jQueryService(neo4jOptions, new Neo4jQueryWriteStrategy(neo4j, SaveMode.Overwrite)).createQuery()\n\n assertEquals(\n s\"\"\"${prefix}UNWIND $$events AS event\n |MERGE (node:Location {name: event.keys.name, type: event.keys.type, featureId: event.keys.featureId})\n |SET node += event.properties\n |\"\"\".stripMargin,\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testCompoundKeysForRelationship(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"BOUGHT\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.source.node.keys\", \"FirstName:name,LastName:lastName\")\n options.put(\"relationship.target.labels\", \"Product\")\n options.put(\"relationship.target.node.keys\", \"ProductPrice:price,ProductId:id\")\n\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String =\n new Neo4jQueryService(neo4jOptions, new Neo4jQueryWriteStrategy(neo4j, SaveMode.Overwrite)).createQuery()\n\n assertEquals(\n s\"\"\"${prefix}UNWIND $$events AS event\n |MATCH (source:Person {name: event.source.keys.name, lastName: event.source.keys.lastName})\n |MATCH (target:Product {price: event.target.keys.price, id: event.target.keys.id})\n |MERGE (source)-[rel:BOUGHT]->(target)\n |SET rel += event.rel.properties\n |\"\"\".stripMargin,\n query.stripMargin\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testCompoundKeysForRelationshipMergeMatch(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"BOUGHT\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.source.node.keys\", \"FirstName:name,LastName:lastName\")\n options.put(\"relationship.source.save.mode\", \"Overwrite\")\n options.put(\"relationship.target.labels\", \"Product\")\n options.put(\"relationship.target.node.keys\", \"ProductPrice:price,ProductId:id\")\n options.put(\"relationship.target.save.mode\", \"match\")\n\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String =\n new Neo4jQueryService(neo4jOptions, new Neo4jQueryWriteStrategy(neo4j, SaveMode.Overwrite)).createQuery()\n\n assertEquals(\n s\"\"\"${prefix}UNWIND $$events AS event\n |MERGE (source:Person {name: event.source.keys.name, lastName: event.source.keys.lastName}) SET source += event.source.properties\n |WITH source, event\n |MATCH (target:Product {price: event.target.keys.price, id: event.target.keys.id})\n |MERGE (source)-[rel:BOUGHT]->(target)\n |SET rel += event.rel.properties\n |\"\"\".stripMargin,\n query.stripMargin\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testShouldDoSumAggregationOnLabels(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val ageField = new DummyNamedReference(\"age\")\n var query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty[Filter],\n PartitionPagination.EMPTY,\n Seq(\"name\", \"SUM(DISTINCT age)\", \"SUM(age)\"),\n Array(\n new Sum(ageField, false),\n new Sum(ageField, true)\n )\n )\n ).createQuery()\n\n assertEquals(\n s\"${prefix}MATCH (n:`Person`) RETURN n.name AS name, sum(DISTINCT n.age) AS `SUM(DISTINCT age)`, sum(n.age) AS `SUM(age)`\",\n query\n )\n\n val nameField = new DummyNamedReference(\"name\")\n query = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty[Filter],\n PartitionPagination.EMPTY,\n Seq(\"name\", \"COUNT(DISTINCT name)\", \"COUNT(name)\"),\n Array(\n new Count(nameField, false),\n new Count(nameField, true)\n )\n )\n ).createQuery()\n\n assertEquals(\n s\"${prefix}MATCH (n:`Person`) RETURN n.name AS name, count(DISTINCT n.name) AS `COUNT(DISTINCT name)`, count(n.name) AS `COUNT(name)`\",\n query\n )\n\n query = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty[Filter],\n PartitionPagination.EMPTY,\n Seq(\"name\", \"MAX(age)\", \"MIN(age)\"),\n Array(\n new Max(ageField),\n new Min(ageField)\n )\n )\n ).createQuery()\n\n assertEquals(\n s\"${prefix}MATCH (n:`Person`) RETURN n.name AS name, max(n.age) AS `MAX(age)`, min(n.age) AS `MIN(age)`\",\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testShouldDoSumAggregationOnRelationships(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"BOUGHT\")\n options.put(\"relationship.nodes.map\", \"false\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Product\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val targetPriceField = new DummyNamedReference(\"`target.price`\")\n var query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty,\n PartitionPagination.EMPTY,\n List(\"source.fullName\", \"SUM(DISTINCT `target.price`)\", \"SUM(`target.price`)\"),\n Array(\n new Sum(targetPriceField, false),\n new Sum(targetPriceField, true)\n )\n )\n ).createQuery()\n\n assertEquals(\n s\"\"\"${prefix}MATCH (source:`Person`)\n |MATCH (target:`Product`)\n |MATCH (source)-[rel:`BOUGHT`]->(target)\n |RETURN source.fullName AS `source.fullName`, sum(DISTINCT target.price) AS `SUM(DISTINCT ``target.price``)`, sum(target.price) AS `SUM(``target.price``)`\"\"\"\n .stripMargin\n .replaceAll(\"\\n\", \" \"),\n query\n )\n\n val targetIdField = new DummyNamedReference(\"`target.id`\")\n query = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty,\n PartitionPagination.EMPTY,\n List(\"source.fullName\", \"COUNT(DISTINCT `target.id`)\", \"COUNT(`target.id`)\"),\n Array(\n new Count(targetIdField, false),\n new Count(targetIdField, true)\n )\n )\n ).createQuery()\n\n assertEquals(\n s\"\"\"${prefix}MATCH (source:`Person`) MATCH (target:`Product`)\n |MATCH (source)-[rel:`BOUGHT`]->(target)\n |RETURN source.fullName AS `source.fullName`, count(DISTINCT target.id) AS `COUNT(DISTINCT ``target.id``)`, count(target.id) AS `COUNT(``target.id``)`\"\"\"\n .stripMargin\n .replaceAll(\"\\n\", \" \"),\n query\n )\n\n query = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty,\n PartitionPagination.EMPTY,\n List(\"source.fullName\", \"MAX(`target.price`)\", \"MIN(`target.price`)\"),\n Array(\n new Max(targetPriceField),\n new Min(targetPriceField)\n )\n )\n ).createQuery()\n\n assertEquals(\n s\"\"\"${prefix}MATCH (source:`Person`)\n |MATCH (target:`Product`)\n |MATCH (source)-[rel:`BOUGHT`]->(target)\n |RETURN source.fullName AS `source.fullName`, max(target.price) AS `MAX(``target.price``)`, min(target.price) AS `MIN(``target.price``)`\"\"\"\n .stripMargin\n .replaceAll(\"\\n\", \" \"),\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testTopNForLabels(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n partitionPagination = PartitionPagination(\n 0,\n 0,\n TopN(\n 42,\n Array(new SortOrder {\n override def expression(): Expression = new DummyNamedReference(\"name\")\n\n override def direction(): SortDirection = SortDirection.ASCENDING\n\n override def nullOrdering(): NullOrdering = direction().defaultNullOrdering()\n })\n )\n )\n )\n ).createQuery()\n\n assertEquals(s\"${prefix}MATCH (n:`Person`) RETURN n ORDER BY n.name ASC LIMIT 42\", query)\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testTopNForLabelsWithRequiredColumn(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n requiredColumns = Array(\"name\"),\n partitionPagination = PartitionPagination(\n 0,\n 0,\n TopN(\n 42,\n Array(new SortOrder {\n override def expression(): Expression = new DummyNamedReference(\"name\")\n\n override def direction(): SortDirection = SortDirection.ASCENDING\n\n override def nullOrdering(): NullOrdering = direction().defaultNullOrdering()\n })\n )\n )\n )\n ).createQuery()\n\n assertEquals(s\"${prefix}MATCH (n:`Person`) RETURN n.name AS name ORDER BY n.name ASC LIMIT 42\", query)\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testTopNForRelationships(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"KNOWS\")\n options.put(\"relationship.nodes.map\", \"false\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty[Filter],\n PartitionPagination(\n 0,\n 0,\n TopN(\n 24,\n Array(new SortOrder {\n override def expression(): Expression = new DummyNamedReference(\"rel.since\")\n\n override def direction(): SortDirection = SortDirection.DESCENDING\n\n override def nullOrdering(): NullOrdering = direction().defaultNullOrdering()\n })\n )\n )\n )\n ).createQuery()\n\n assertEquals(\n s\"${prefix}MATCH (source:`Person`) \" +\n \"MATCH (target:`Person`) \" +\n \"MATCH (source)-[rel:`KNOWS`]->(target) RETURN rel, source AS source, target AS target \" +\n \"ORDER BY rel.since DESC LIMIT 24\",\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testTopNForRelationshipWithOneRequiredColumn(neo4j: Neo4j, prefix: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"relationship\", \"KNOWS\")\n options.put(\"relationship.nodes.map\", \"false\")\n options.put(\"relationship.source.labels\", \"Person\")\n options.put(\"relationship.target.labels\", \"Person\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty[Filter],\n PartitionPagination(\n 0,\n 0,\n TopN(\n 24,\n Array(new SortOrder {\n override def expression(): Expression = new DummyNamedReference(\"rel.since\")\n\n override def direction(): SortDirection = SortDirection.DESCENDING\n\n override def nullOrdering(): NullOrdering = direction().defaultNullOrdering()\n })\n )\n ),\n Array(\"source.name\")\n )\n ).createQuery()\n\n assertEquals(\n s\"\"\"${prefix}MATCH (source:`Person`)\n |MATCH (target:`Person`)\n |MATCH (source)-[rel:`KNOWS`]->(target) RETURN source.name AS `source.name`\n |ORDER BY rel.since DESC LIMIT 24\"\"\"\n .stripMargin\n .replaceAll(\"\\n\", \" \"),\n query\n )\n }\n\n @Test\n @Parameters(method = \"versions_and_prefixes\")\n def testTopNForCustomQueryIgnoresAggregation(neo4j: Neo4j, ignored: String): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.QUERY.toString.toLowerCase, \"MATCH (p:Person) RETURN p\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val query: String = new Neo4jQueryService(\n neo4jOptions,\n new Neo4jQueryReadStrategy(\n neo4j,\n Array.empty[Filter],\n PartitionPagination(\n 0,\n 0,\n TopN(\n 24,\n Array(new SortOrder {\n override def expression(): Expression = new DummyNamedReference(\"name\")\n\n override def direction(): SortDirection = SortDirection.DESCENDING\n\n override def nullOrdering(): NullOrdering = direction().defaultNullOrdering()\n })\n )\n )\n )\n ).createQuery()\n\n assertEquals(\n \"WITH $scriptResult AS scriptResult MATCH (p:Person) RETURN p SKIP 0 LIMIT 24\",\n query\n )\n }\n\n def versions_and_prefixes(): Array[Array[Any]] = {\n Array(\n Array(neo4j(version(4, 4), COMMUNITY), \"\"),\n Array(neo4j(version(4, 4), ENTERPRISE), \"\"),\n Array(neo4j(version(5, 0), COMMUNITY), \"\"),\n Array(neo4j(version(5, 0), ENTERPRISE), \"\"),\n Array(neo4j(version(5, 21), COMMUNITY), \"CYPHER 5 \"),\n Array(neo4j(version(5, 21), ENTERPRISE), \"CYPHER 5 \"),\n Array(neo4j(version(5, 26), COMMUNITY), \"CYPHER 5 \"),\n Array(neo4j(version(5, 26), ENTERPRISE), \"CYPHER 5 \"),\n Array(neo4j(version(2025, 1), COMMUNITY), \"CYPHER 5 \"),\n Array(neo4j(version(2025, 1), ENTERPRISE), \"CYPHER 5 \")\n )\n }\n\n def neo4j(version: Neo4jVersion, edition: Neo4jEdition): Neo4j = {\n new Neo4j(version, edition, SELF_MANAGED)\n }\n\n def version(major: Int, minor: Int): Neo4jVersion = {\n new Neo4jVersion(major, minor, 0)\n }\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/service/SchemaServiceTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.service\n\nimport org.apache.spark.sql.types.DataTypes\nimport org.apache.spark.sql.types.StructField\nimport org.apache.spark.sql.types.StructType\nimport org.junit.Assert._\nimport org.junit.Before\nimport org.junit.FixMethodOrder\nimport org.junit.Test\nimport org.junit.runners.MethodSorters\nimport org.neo4j.Closeables.use\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.caniuse.Neo4jDeploymentType\nimport org.neo4j.caniuse.Neo4jEdition\nimport org.neo4j.caniuse.Neo4jVersion\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.summary.ResultSummary\nimport org.neo4j.spark.SparkConnectorScalaBaseTSE\nimport org.neo4j.spark.SparkConnectorScalaSuiteIT\nimport org.neo4j.spark.SparkConnectorScalaSuiteIT.neo4j\nimport org.neo4j.spark.converter.CypherToSparkTypeConverter\nimport org.neo4j.spark.util.DriverCache\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.Neo4jUtil\nimport org.neo4j.spark.util.QueryType\n\nimport java.util\nimport java.util.UUID\n\n@FixMethodOrder(MethodSorters.JVM)\nclass SchemaServiceTSE extends SparkConnectorScalaBaseTSE {\n\n @Before\n def beforeEach(): Unit = {\n use(SparkConnectorScalaSuiteIT.session(\"system\")) {\n session =>\n session.run(\"CREATE OR REPLACE DATABASE neo4j WAIT 30 seconds\")\n .consume()\n }\n }\n\n @Test\n def testGetSchemaFromNodeBoolean(): Unit = {\n initTest(\"CREATE (p:Person {is_hero: true})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"is_hero\", DataTypes.BooleanType))), schema)\n }\n\n @Test\n def testGetSchemaFromNodeString(): Unit = {\n initTest(\"CREATE (p:Person {name: 'John'})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"name\", DataTypes.StringType))), schema)\n }\n\n @Test\n def testGetSchemaFromNodeLong(): Unit = {\n initTest(\"CREATE (p:Person {age: 93})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"age\", DataTypes.LongType))), schema)\n }\n\n @Test\n def testGetSchemaFromNodeDouble(): Unit = {\n initTest(\"CREATE (p:Person {ratio: 43.120})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"ratio\", DataTypes.DoubleType))), schema)\n }\n\n @Test\n def testGetSchemaFromNodePoint2D(): Unit = {\n initTest(\"CREATE (p:Person {location: point({x: 12.32, y: 49.32})})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"location\", CypherToSparkTypeConverter.pointType))), schema)\n }\n\n @Test\n def testGetSchemaFromDate(): Unit = {\n initTest(\"CREATE (p:Person {born_on: date('1998-01-05')})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"born_on\", DataTypes.DateType))), schema)\n }\n\n @Test\n def testGetSchemaFromDateTime(): Unit = {\n initTest(\"CREATE (p:Person {arrived_at: datetime('1998-01-05')})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"arrived_at\", DataTypes.TimestampType))), schema)\n }\n\n @Test\n def testGetSchemaFromTime(): Unit = {\n initTest(\"CREATE (p:Person {arrived_at: time('125035.556+0100')})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"arrived_at\", CypherToSparkTypeConverter.timeType))), schema)\n }\n\n @Test\n def testGetSchemaFromStringArray(): Unit = {\n initTest(\"CREATE (p:Person {names: ['John', 'Doe']})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(\n getExpectedStructType(Seq(StructField(\"names\", DataTypes.createArrayType(DataTypes.StringType)))),\n schema\n )\n }\n\n @Test\n def testGetSchemaFromDateArray(): Unit = {\n initTest(\"CREATE (p:Person {names: [date('2019-11-19'), date('2019-11-20')]})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(\n getExpectedStructType(Seq(StructField(\"names\", DataTypes.createArrayType(DataTypes.DateType)))),\n schema\n )\n }\n\n @Test\n def testGetSchemaFromTimestampArray(): Unit = {\n initTest(\"CREATE (p:Person {dates: [datetime('2019-11-19'), datetime('2019-11-20')]})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(\n getExpectedStructType(Seq(StructField(\"dates\", DataTypes.createArrayType(DataTypes.TimestampType)))),\n schema\n )\n }\n\n @Test\n def testGetSchemaFromTimeArray(): Unit = {\n initTest(\"CREATE (p:Person {dates: [time('125035.556+0100'), time('125125.556+0100')]})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(\n getExpectedStructType(Seq(StructField(\"dates\", DataTypes.createArrayType(CypherToSparkTypeConverter.timeType)))),\n schema\n )\n }\n\n @Test\n def testGetSchemaFromIntegerArray(): Unit = {\n initTest(\"CREATE (p:Person {ages: [42, 101]})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"ages\", DataTypes.createArrayType(DataTypes.LongType)))), schema)\n }\n\n @Test\n def testGetSchemaFromMultipleNodes(): Unit = {\n initTest(\n \"\"\"\n CREATE (p1:Person {age: 31, name: 'Jane Doe'}),\n (p2:Person {name: 'John Doe', age: 33, location: null}),\n (p3:Person {age: 25, location: point({latitude: 12.12, longitude: 31.13})})\n \"\"\"\n )\n\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(\n getExpectedStructType(Seq(\n StructField(\"age\", DataTypes.LongType),\n StructField(\"location\", CypherToSparkTypeConverter.pointType),\n StructField(\"name\", DataTypes.StringType)\n )),\n schema\n )\n }\n\n private def getExpectedStructType(structFields: Seq[StructField]): StructType = {\n val additionalFields: Seq[StructField] = Seq(\n StructField(Neo4jUtil.INTERNAL_LABELS_FIELD, DataTypes.createArrayType(DataTypes.StringType), nullable = true),\n StructField(Neo4jUtil.INTERNAL_ID_FIELD, DataTypes.LongType, nullable = false)\n )\n StructType(structFields.union(additionalFields).reverse)\n }\n\n private def initTest(query: String): Unit = {\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(query).consume()\n }\n )\n }\n\n private def getSchema(options: java.util.Map[String, String]): StructType = {\n options.put(Neo4jOptions.URL, SparkConnectorScalaSuiteIT.server.getBoltUrl)\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val driverCache = new DriverCache(neo4jOptions.connection)\n val schemaService: SchemaService = new SchemaService(neo4j, neo4jOptions, driverCache)\n\n val schema: StructType = schemaService.struct()\n schemaService.close()\n driverCache.close()\n\n schema\n }\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/service/SchemaServiceTest.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.service\n\nimport org.junit.Assert.assertEquals\nimport org.junit.Test\nimport org.mockito.Mockito.RETURNS_DEEP_STUBS\nimport org.mockito.Mockito.mock\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.caniuse.Neo4jDeploymentType\nimport org.neo4j.caniuse.Neo4jEdition\nimport org.neo4j.caniuse.Neo4jVersion\nimport org.neo4j.spark.config.TopN\nimport org.neo4j.spark.util.DriverCache\nimport org.neo4j.spark.util.Neo4jOptions\n\nimport scala.annotation.nowarn\nimport scala.collection.JavaConverters\n\nclass SchemaServiceTest {\n\n @Test\n def does_not_overflow_when_partition_size_is_over_max_value_of_32bit_integers(): Unit = {\n // note: _ cannot be used to separate digit groups, as this requires Scala 2.13+\n val opts = options(\n \"url\" -> \"bolt://example.com\",\n \"partitions\" -> 2.toString,\n \"query.count\" -> (2L * 2147483648L).toString, // 2 * (Integer.MAX_VALUE + 1)\n \"query\" -> \"MERGE (:Node)\"\n )\n val schemaService = new SchemaService(neo4j(), opts, mock(classOf[DriverCache], RETURNS_DEEP_STUBS))\n\n val pages = schemaService.skipLimitFromPartition(Some(TopN(1024)))\n\n assertEquals(List(0, 1), pages.map(_.partitionNumber).toList)\n assertEquals(List(0, 2147483648L), pages.map(_.skip).toList)\n assertEquals(List(2147483648L, 2147483648L), pages.map(_.topN.limit).toList)\n assertEquals(List(0, 0), pages.map(_.topN.orders.size).toList)\n }\n\n private def options(kv: (String, String)*): Neo4jOptions = {\n new Neo4jOptions(\n JavaConverters.mapAsJavaMap(kv.toMap)\n )\n }\n\n private def neo4j(): Neo4j = {\n new Neo4j(new Neo4jVersion(2025, 1, 0), Neo4jEdition.COMMUNITY, Neo4jDeploymentType.SELF_MANAGED)\n }\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/service/SchemaServiceWithApocTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.service\n\nimport org.apache.spark.sql.types.DataTypes\nimport org.apache.spark.sql.types.StructField\nimport org.apache.spark.sql.types.StructType\nimport org.junit.Assert._\nimport org.junit.Before\nimport org.junit.FixMethodOrder\nimport org.junit.Test\nimport org.junit.runners.MethodSorters\nimport org.neo4j.Closeables.use\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.caniuse.Neo4jDeploymentType\nimport org.neo4j.caniuse.Neo4jEdition\nimport org.neo4j.caniuse.Neo4jVersion\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.summary.ResultSummary\nimport org.neo4j.spark.SparkConnectorScalaBaseWithApocTSE\nimport org.neo4j.spark.SparkConnectorScalaSuiteWithApocIT\nimport org.neo4j.spark.converter.CypherToSparkTypeConverter\nimport org.neo4j.spark.util.DriverCache\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.Neo4jUtil\nimport org.neo4j.spark.util.QueryType\n\nimport java.util\nimport java.util.UUID\n\n@FixMethodOrder(MethodSorters.JVM)\nclass SchemaServiceWithApocTSE extends SparkConnectorScalaBaseWithApocTSE {\n\n @Before\n def beforeEach(): Unit = {\n use(SparkConnectorScalaSuiteWithApocIT.session(\"system\")) {\n session =>\n session.run(\"CREATE OR REPLACE DATABASE neo4j WAIT 30 seconds\")\n .consume()\n }\n }\n\n @Test\n def testGetSchemaFromNodeBoolean(): Unit = {\n initTest(\"CREATE (p:Person {is_hero: true})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"is_hero\", DataTypes.BooleanType))), schema)\n }\n\n @Test\n def testGetSchemaFromNodeString(): Unit = {\n initTest(\"CREATE (p:Person {name: 'John'})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"name\", DataTypes.StringType))), schema)\n }\n\n @Test\n def testGetSchemaFromNodeLong(): Unit = {\n initTest(\"CREATE (p:Person {age: 93})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"age\", DataTypes.LongType))), schema)\n }\n\n @Test\n def testGetSchemaFromNodeDouble(): Unit = {\n initTest(\"CREATE (p:Person {ratio: 43.120})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"ratio\", DataTypes.DoubleType))), schema)\n }\n\n @Test\n def testGetSchemaFromNodePoint2D(): Unit = {\n initTest(\"CREATE (p:Person {location: point({x: 12.32, y: 49.32})})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"location\", CypherToSparkTypeConverter.pointType))), schema)\n }\n\n @Test\n def testGetSchemaFromDate(): Unit = {\n initTest(\"CREATE (p:Person {born_on: date('1998-01-05')})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"born_on\", DataTypes.DateType))), schema)\n }\n\n @Test\n def testGetSchemaFromDateTime(): Unit = {\n initTest(\"CREATE (p:Person {arrived_at: datetime('1998-01-05')})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"arrived_at\", DataTypes.TimestampType))), schema)\n }\n\n @Test\n def testGetSchemaFromTime(): Unit = {\n initTest(\"CREATE (p:Person {arrived_at: time('125035.556+0100')})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"arrived_at\", CypherToSparkTypeConverter.timeType))), schema)\n }\n\n @Test\n def testGetSchemaFromStringArray(): Unit = {\n initTest(\"CREATE (p:Person {names: ['John', 'Doe']})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(\n getExpectedStructType(Seq(StructField(\"names\", DataTypes.createArrayType(DataTypes.StringType)))),\n schema\n )\n }\n\n @Test\n def testGetSchemaFromDateArray(): Unit = {\n initTest(\"CREATE (p:Person {names: [date('2019-11-19'), date('2019-11-20')]})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(\n getExpectedStructType(Seq(StructField(\"names\", DataTypes.createArrayType(DataTypes.DateType)))),\n schema\n )\n }\n\n @Test\n def testGetSchemaFromTimestampArray(): Unit = {\n initTest(\"CREATE (p:Person {dates: [datetime('2019-11-19'), datetime('2019-11-20')]})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(\n getExpectedStructType(Seq(StructField(\"dates\", DataTypes.createArrayType(DataTypes.TimestampType)))),\n schema\n )\n }\n\n @Test\n def testGetSchemaFromTimeArray(): Unit = {\n initTest(\"CREATE (p:Person {dates: [time('125035.556+0100'), time('125125.556+0100')]})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(\n getExpectedStructType(Seq(StructField(\"dates\", DataTypes.createArrayType(CypherToSparkTypeConverter.timeType)))),\n schema\n )\n }\n\n @Test\n def testGetSchemaFromIntegerArray(): Unit = {\n initTest(\"CREATE (p:Person {ages: [42, 101]})\")\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(getExpectedStructType(Seq(StructField(\"ages\", DataTypes.createArrayType(DataTypes.LongType)))), schema)\n }\n\n @Test\n def testGetSchemaFromMultipleNodes(): Unit = {\n initTest(\n \"\"\"\n CREATE (p1:Person {age: 31, name: 'Jane Doe'}),\n (p2:Person {name: 'John Doe', age: 33, location: null}),\n (p3:Person {age: 25, location: point({latitude: 12.12, longitude: 31.13})})\n \"\"\"\n )\n\n val options: java.util.Map[String, String] = new util.HashMap[String, String]()\n options.put(QueryType.LABELS.toString.toLowerCase, \"Person\")\n\n val schema = getSchema(options)\n\n assertEquals(\n getExpectedStructType(Seq(\n StructField(\"age\", DataTypes.LongType),\n StructField(\"location\", CypherToSparkTypeConverter.pointType),\n StructField(\"name\", DataTypes.StringType)\n )),\n schema\n )\n }\n\n private def getExpectedStructType(structFields: Seq[StructField]): StructType = {\n val additionalFields: Seq[StructField] = Seq(\n StructField(Neo4jUtil.INTERNAL_LABELS_FIELD, DataTypes.createArrayType(DataTypes.StringType), nullable = true),\n StructField(Neo4jUtil.INTERNAL_ID_FIELD, DataTypes.LongType, nullable = false)\n )\n StructType(structFields.union(additionalFields).reverse)\n }\n\n private def initTest(query: String): Unit = {\n SparkConnectorScalaSuiteWithApocIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(query).consume()\n }\n )\n }\n\n private def getSchema(options: java.util.Map[String, String]): StructType = {\n options.put(Neo4jOptions.URL, SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val driverCache = new DriverCache(neo4jOptions.connection)\n val neo4j = new Neo4j(new Neo4jVersion(4, 4, 0), Neo4jEdition.ENTERPRISE, Neo4jDeploymentType.SELF_MANAGED)\n val schemaService: SchemaService = new SchemaService(neo4j, neo4jOptions, driverCache)\n\n val schema: StructType = schemaService.struct()\n schemaService.close()\n driverCache.close()\n\n schema\n }\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/util/DummyNamedReference.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport org.apache.spark.sql.connector.expressions.NamedReference\n\nclass DummyNamedReference(private val fields: String*) extends NamedReference {\n override def fieldNames(): Array[String] = fields.toArray\n\n override def describe(): String = fields.mkString(\", \")\n override def toString: String = describe()\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/util/Neo4jImplicitsTest.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport org.apache.spark.sql.connector.expressions.NamedReference\nimport org.apache.spark.sql.connector.expressions.aggregate.Aggregation\nimport org.apache.spark.sql.connector.expressions.aggregate.Sum\nimport org.apache.spark.sql.sources.And\nimport org.apache.spark.sql.sources.EqualTo\nimport org.apache.spark.sql.types.DataTypes\nimport org.apache.spark.sql.types.StructField\nimport org.apache.spark.sql.types.StructType\nimport org.junit.Assert\nimport org.junit.Assert._\nimport org.junit.Test\nimport org.neo4j.spark.util.Neo4jImplicits._\n\nimport scala.collection.JavaConverters.mapAsJavaMapConverter\nimport scala.collection.JavaConverters.mapAsScalaMapConverter\nimport scala.collection.JavaConverters.seqAsJavaListConverter\nimport scala.collection.immutable.ListMap\n\nclass Neo4jImplicitsTest {\n\n @Test\n def `should quote the string` {\n // given\n val value = \"Test with space\"\n\n // when\n val actual = value.quote\n\n // then\n assertEquals(s\"`$value`\", actual)\n }\n\n @Test\n def `should quote text that starts with $` {\n // given\n val value = \"$tring\"\n\n // when\n val actual = value.quote\n\n // then\n assertEquals(s\"`$value`\", actual)\n }\n\n @Test\n def `should not re-quote the string` {\n // given\n val value = \"`Test with space`\"\n\n // when\n val actual = value.quote\n\n // then\n assertEquals(value, actual)\n }\n\n @Test\n def `should not quote the string` {\n // given\n val value = \"Test\"\n\n // when\n val actual = value.quote\n\n // then\n assertEquals(value, actual)\n }\n\n @Test\n def `should return attribute if filter has it` {\n // given\n val filter = EqualTo(\"name\", \"John\")\n\n // when\n val attribute = filter.getAttribute\n\n // then\n assertTrue(attribute.isDefined)\n }\n\n @Test\n def `should return an empty option if the filter doesn't have an attribute` {\n // given\n val filter = And(EqualTo(\"name\", \"John\"), EqualTo(\"age\", 32))\n\n // when\n val attribute = filter.getAttribute\n\n // then\n assertFalse(attribute.isDefined)\n }\n\n @Test\n def `should return the attribute without the entity identifier` {\n // given\n val filter = EqualTo(\"person.address.coords\", 32)\n\n // when\n val attribute = filter.getAttributeWithoutEntityName\n\n // then\n assertEquals(\"address.coords\", attribute.get)\n }\n\n @Test\n def `struct should return true if contains fields`: Unit = {\n val struct = StructType(Seq(\n StructField(\"is_hero\", DataTypes.BooleanType),\n StructField(\"name\", DataTypes.StringType),\n StructField(\"fi``(╯°□°)╯︵ ┻━┻eld\", DataTypes.StringType)\n ))\n\n assertEquals(0, struct.getMissingFields(Set(\"is_hero\", \"name\", \"fi``(╯°□°)╯︵ ┻━┻eld\")).size)\n }\n\n @Test\n def `struct should return false if not contains fields`: Unit = {\n val struct =\n StructType(Seq(StructField(\"is_hero\", DataTypes.BooleanType), StructField(\"name\", DataTypes.StringType)))\n\n assertEquals(Set[String](\"hero_name\"), struct.getMissingFields(Set(\"is_hero\", \"hero_name\")))\n }\n\n @Test\n def `getMissingFields should handle maps`: Unit = {\n val struct = StructType(Seq(\n StructField(\"im\", DataTypes.StringType),\n StructField(\"im.a\", DataTypes.createMapType(DataTypes.StringType, DataTypes.StringType)),\n StructField(\"im.also.a\", DataTypes.createMapType(DataTypes.StringType, DataTypes.StringType)),\n StructField(\"im.not.a.map\", DataTypes.StringType),\n StructField(\"fi``(╯°□°)╯︵ ┻━┻eld\", DataTypes.StringType)\n ))\n\n val result = struct.getMissingFields(Set(\n \"im.aMap\",\n \"`im.also.a`.field\",\n \"`im.a`.map\",\n \"`im.not.a.map`\",\n \"fi``(╯°□°)╯︵ ┻━┻eld\"\n ))\n\n assertEquals(Set(\"im.aMap\"), result)\n }\n\n @Test\n def `groupByCols aggregation should work`: Unit = {\n val aggField = new NamedReference {\n override def fieldNames(): Array[String] = Array(\"foo\")\n\n override def describe(): String = \"foo\"\n }\n val gbyField = new NamedReference {\n override def fieldNames(): Array[String] = Array(\"bar\")\n\n override def describe(): String = \"bar\"\n }\n val agg = new Aggregation(Array(new Sum(aggField, false)), Array(gbyField))\n assertEquals(1, agg.groupByCols().length)\n assertEquals(\"bar\", agg.groupByCols()(0).describe())\n }\n\n @Test\n def `should flatten the map`(): Unit = {\n val input = Map(\n \"foo\" -> \"bar\",\n \"key\" -> Map(\n \"innerKey\" -> Map(\"innerKey2\" -> \"value\")\n )\n )\n val expected = Map(\n \"foo\" -> \"bar\",\n \"key.innerKey.innerKey2\" -> \"value\"\n )\n val actual = input.flattenMap()\n Assert.assertEquals(expected, actual)\n }\n\n @Test\n def `should not handle collision`(): Unit = {\n val input = ListMap(\n \"my\" -> Map(\n \"inner\" -> Map(\"key\" -> 42424242),\n \"inner.key\" -> 424242\n ),\n \"my.inner\" -> Map(\"key\" -> 4242).asJava,\n \"my.inner.key\" -> 42\n )\n val expected = Map(\n \"my.inner.key\" -> 42\n )\n val actual = input.flattenMap()\n Assert.assertEquals(expected, actual)\n }\n\n @Test\n def `should handle collision by aggregating values`(): Unit = {\n val input = ListMap(\n \"my\" -> Map(\n \"inner\" -> Map(\"key\" -> 42424242),\n \"inner.key\" -> 424242\n ),\n \"my.inner\" -> Map(\"key\" -> 4242).asJava,\n \"my.inner.key\" -> 42\n )\n val expected = Map(\n \"my.inner.key\" -> Seq(42424242, 424242, 4242, 42).asJava\n )\n val actual = input.flattenMap(groupDuplicateKeys = true)\n Assert.assertEquals(expected, actual)\n }\n\n @Test\n def `should show duplicate keys`(): Unit = {\n val input = Map(\n \"my\" -> Map(\n \"inner\" -> Map(\"key\" -> 42424242),\n \"inner.key\" -> 424242\n ),\n \"my.inner\" -> Map(\"key\" -> 4242).asJava,\n \"my.inner.key\" -> 42\n )\n val expected = Seq(\"my.inner.key\", \"my.inner.key\", \"my.inner.key\", \"my.inner.key\")\n val actual = input.flattenKeys()\n Assert.assertEquals(expected, actual)\n }\n\n @Test\n def `should deserialized dotted/stringified map into a nested Java map`(): Unit = {\n val actual = Map(\n \"graphName\" -> \"foo\",\n \"configuration.number\" -> \"1\",\n \"configuration.string\" -> \"foo\",\n \"configuration.list\" -> \"['a', 1]\",\n \"configuration.map.key\" -> \"value\",\n \"relationshipProjection.LINK.properties.foobar.defaultValue\" -> \"42.0\"\n ).toNestedJavaMap\n val expected: java.util.Map[String, Object] = Map(\n \"graphName\" -> \"foo\",\n \"configuration\" -> Map(\n \"number\" -> 1,\n \"string\" -> \"foo\",\n \"list\" -> Seq(\"a\", 1).toList.asJava,\n \"map\" -> Map(\n \"key\" -> \"value\"\n ).asJava\n ).asJava,\n \"relationshipProjection\" -> Map(\n \"LINK\" -> Map(\n \"properties\" -> Map(\n \"foobar\" -> Map(\"defaultValue\" -> 42.0).asJava\n ).asJava\n ).asJava\n ).asJava\n ).asJava\n Assert.assertEquals(expected, actual)\n\n val ucActual = Map(\n \"graphName\" -> \"myGraph\",\n \"nodeProjection\" -> \"Website\",\n \"relationshipProjection.LINK.indexInverse\" -> \"true\"\n ).toNestedJavaMap\n val ucExpected: java.util.Map[String, Object] = Map(\n \"graphName\" -> \"myGraph\",\n \"nodeProjection\" -> \"Website\",\n \"relationshipProjection\" -> Map(\n \"LINK\" -> Map(\n \"indexInverse\" -> true\n ).asJava\n ).asJava\n ).asJava\n Assert.assertEquals(ucExpected, ucActual)\n }\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/util/Neo4jOptionsIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport org.junit.Assert.assertEquals\nimport org.junit.Assert.assertNotNull\nimport org.junit.Ignore\nimport org.junit.Test\nimport org.neo4j.Closeables.use\nimport org.neo4j.spark.SparkConnectorScalaSuiteIT\nimport org.neo4j.spark.SparkConnectorScalaSuiteIT.server\n\nclass Neo4jOptionsIT extends SparkConnectorScalaSuiteIT {\n\n @Test\n def shouldConstructDriver(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, server.getBoltUrl)\n options.put(Neo4jOptions.AUTH_TYPE, \"none\")\n\n val neo4jOptions = new Neo4jOptions(options)\n\n use(neo4jOptions.connection.createDriver()) { driver =>\n assertNotNull(driver)\n\n use(driver.session()) { session =>\n assertEquals(1, session.run(\"RETURN 1\").single().get(0).asInt())\n }\n }\n }\n\n @Test\n def shouldConstructDriverWithResolver(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(\n Neo4jOptions.URL,\n s\"neo4j://localhost.localdomain:8888, bolt://localhost.localdomain:9999, ${server.getBoltUrl}\"\n )\n options.put(Neo4jOptions.AUTH_TYPE, \"none\")\n\n val neo4jOptions = new Neo4jOptions(options)\n\n use(neo4jOptions.connection.createDriver()) { driver =>\n assertNotNull(driver)\n\n use(driver.session()) { session =>\n assertEquals(1, session.run(\"RETURN 1\").single().get(0).asInt())\n }\n }\n }\n\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/util/Neo4jOptionsTest.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport org.junit.Assert._\nimport org.junit.Test\nimport org.neo4j.driver.AccessMode\nimport org.neo4j.driver.net.ServerAddress\n\nimport java.net.URI\nimport java.time.Duration\n\nimport scala.annotation.meta.getter\nimport scala.collection.JavaConverters._\n\nclass Neo4jOptionsTest {\n\n import org.junit.Rule\n import org.junit.rules.ExpectedException\n\n @(Rule @getter)\n val _expectedException: ExpectedException = ExpectedException.none\n\n @Test\n def testUrlIsRequired(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(QueryType.QUERY.toString.toLowerCase, \"Person\")\n\n _expectedException.expect(classOf[IllegalArgumentException])\n _expectedException.expectMessage(\"Parameter 'url' is required\")\n\n new Neo4jOptions(options)\n }\n\n @Test\n def testRelationshipTableName(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.RELATIONSHIP.toString.toLowerCase, \"KNOWS\")\n options.put(Neo4jOptions.RELATIONSHIP_SOURCE_LABELS, \"Person\")\n options.put(Neo4jOptions.RELATIONSHIP_TARGET_LABELS, \"Answer\")\n\n val neo4jOptions = new Neo4jOptions(options)\n\n assertEquals(\"table_Person_KNOWS_Answer\", neo4jOptions.getTableName)\n }\n\n @Test\n def testLabelsTableName(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"labels\", \"Person:Admin\")\n\n val neo4jOptions = new Neo4jOptions(options)\n\n assertEquals(\"table_Person-Admin\", neo4jOptions.getTableName)\n }\n\n @Test\n def testRelationshipNodeModesAreCaseInsensitive(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.RELATIONSHIP.toString.toLowerCase, \"KNOWS\")\n options.put(Neo4jOptions.RELATIONSHIP_SAVE_STRATEGY, \"nAtIve\")\n options.put(Neo4jOptions.RELATIONSHIP_SOURCE_SAVE_MODE, \"Errorifexists\")\n options.put(Neo4jOptions.RELATIONSHIP_TARGET_SAVE_MODE, \"overwrite\")\n\n val neo4jOptions = new Neo4jOptions(options)\n\n assertEquals(RelationshipSaveStrategy.NATIVE, neo4jOptions.relationshipMetadata.saveStrategy)\n assertEquals(NodeSaveMode.ErrorIfExists, neo4jOptions.relationshipMetadata.sourceSaveMode)\n assertEquals(NodeSaveMode.Overwrite, neo4jOptions.relationshipMetadata.targetSaveMode)\n }\n\n @Test\n def testRelationshipWriteStrategyIsNotPresentShouldThrowException(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, \"PERSON\")\n options.put(\"relationship.save.strategy\", \"nope\")\n\n _expectedException.expect(classOf[NoSuchElementException])\n _expectedException.expectMessage(\"No value found for 'NOPE'\")\n\n new Neo4jOptions(options)\n }\n\n @Test\n def testQueryShouldHaveQueryType(): Unit = {\n val query: String = \"MATCH n RETURN n\"\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.QUERY.toString.toLowerCase, query)\n\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n assertEquals(QueryType.QUERY, neo4jOptions.query.queryType)\n assertEquals(query, neo4jOptions.query.value)\n }\n\n @Test\n def testNodeShouldHaveLabelType(): Unit = {\n val label: String = \"Person\"\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, label)\n\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n assertEquals(QueryType.LABELS, neo4jOptions.query.queryType)\n assertEquals(label, neo4jOptions.query.value)\n }\n\n @Test\n def testRelationshipShouldHaveRelationshipType(): Unit = {\n val relationship: String = \"KNOWS\"\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.LABELS.toString.toLowerCase, relationship)\n\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n assertEquals(QueryType.LABELS, neo4jOptions.query.queryType)\n assertEquals(relationship, neo4jOptions.query.value)\n }\n\n @Test\n def testPushDownColumnIsDisabled(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(\"pushdown.columns.enabled\", \"false\")\n\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n assertFalse(neo4jOptions.pushdownColumnsEnabled)\n }\n\n @Test\n def testDriverDefaults(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n options.put(QueryType.QUERY.toString.toLowerCase, \"MATCH n RETURN n\")\n\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n assertEquals(\"\", neo4jOptions.session.database)\n assertEquals(AccessMode.READ, neo4jOptions.session.accessMode)\n\n assertEquals(\"basic\", neo4jOptions.connection.auth)\n assertEquals(Neo4jOptions.DEFAULT_AUTH_PARAMETERS, neo4jOptions.connection.authParameters)\n assertEquals(false, neo4jOptions.connection.encryption)\n\n assertEquals(None, neo4jOptions.connection.trustStrategy)\n\n assertEquals(\"\", neo4jOptions.connection.certificatePath)\n\n assertEquals(Neo4jOptions.DEFAULT_CONNECTION_MAX_LIFETIME_MSECS, neo4jOptions.connection.lifetime)\n assertEquals(-1, neo4jOptions.connection.acquisitionTimeout)\n assertEquals(-1, neo4jOptions.connection.connectionTimeout)\n assertEquals(\n Neo4jOptions.DEFAULT_CONNECTION_LIVENESS_CHECK_TIMEOUT_MSECS,\n neo4jOptions.connection.livenessCheckTimeout\n )\n assertEquals(RelationshipSaveStrategy.NATIVE, neo4jOptions.relationshipMetadata.saveStrategy)\n\n assertTrue(neo4jOptions.pushdownFiltersEnabled)\n }\n\n @Test\n def testApocConfiguration(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(\"apoc.meta.nodeTypeProperties\", \"\"\"{\"nodeLabels\": [\"Label\"], \"mandatory\": false}\"\"\")\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n val expected = Map(\"apoc.meta.nodeTypeProperties\" -> Map(\n \"nodeLabels\" -> Seq(\"Label\").asJava,\n \"mandatory\" -> false\n ))\n\n assertEquals(neo4jOptions.apocConfig.procedureConfigMap, expected)\n }\n\n @Test\n def testUnexistingProperty(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(\"relationship.properties\", null)\n options.put(Neo4jOptions.URL, \"bolt://localhost\")\n\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n\n assertEquals(neo4jOptions.relationshipMetadata.properties, None)\n }\n\n @Test\n def testUrls(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"neo4j://localhost, neo4j://foo.bar:7687, neo4j://foo.bar.baz:7783\")\n\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n val (baseUrl, resolvers) = neo4jOptions.connection.connectionUrls\n\n assertEquals(URI.create(\"neo4j://localhost\"), baseUrl)\n assertEquals(Set(ServerAddress.of(\"foo.bar\", 7687), ServerAddress.of(\"foo.bar.baz\", 7783)), resolvers)\n }\n\n @Test\n def testGdsProperties(): Unit = {\n val options: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n options.put(Neo4jOptions.URL, \"neo4j://localhost,neo4j://foo.bar,neo4j://foo.bar.baz:7783\")\n options.put(\"gds\", \"gds.pageRank.stream\")\n options.put(\"gds.graphName\", \"myGraph\")\n options.put(\"gds.configuration.concurrency\", \"2\")\n val neo4jOptions: Neo4jOptions = new Neo4jOptions(options)\n assertEquals(QueryType.GDS, neo4jOptions.query.queryType)\n assertEquals(\"gds.pageRank.stream\", neo4jOptions.query.value)\n assertEquals(\n Map(\n \"graphName\" -> \"myGraph\",\n \"configuration\" -> Map(\"concurrency\" -> 2).asJava\n ).asJava,\n neo4jOptions.gdsMetadata.parameters\n )\n }\n\n @Test\n def testTransactionTimeout(): Unit = {\n // Given a Neo4j options with transaction timeout set\n val rawOptions = new java.util.HashMap[String, String]()\n rawOptions.put(Neo4jOptions.URL, \"neo4j://localhost,neo4j://foo.bar,neo4j://foo.bar.baz:7783\")\n rawOptions.put(\"db.transaction.timeout\", \"1000\")\n\n val neo4jOptions = new Neo4jOptions(rawOptions)\n\n // When it converts to TransactionConfig\n val transactionConfig = neo4jOptions.toNeo4jTransactionConfig\n\n // Then it has the correct duration\n assertEquals(Duration.ofMillis(1000), transactionConfig.timeout())\n }\n\n @Test\n def testDefaultTransactionTimeout(): Unit = {\n // Given a Neo4j options with no explicit transaction timeout set\n val rawOptions = new java.util.HashMap[String, String]()\n rawOptions.put(Neo4jOptions.URL, \"neo4j://localhost,neo4j://foo.bar,neo4j://foo.bar.baz:7783\")\n\n val neo4jOptions = new Neo4jOptions(rawOptions)\n\n // When it converts to TransactionConfig\n val transactionConfig = neo4jOptions.toNeo4jTransactionConfig\n\n // Then it is not set\n assertNull(transactionConfig.timeout())\n }\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/util/Neo4jUtilTest.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport org.apache.commons.lang3.StringUtils\nimport org.junit.Assert\nimport org.junit.Test\n\nclass Neo4jUtilTest {\n\n @Test\n def testSafetyCloseShouldNotFailWithNull(): Unit = {\n Neo4jUtil.closeSafely(null)\n }\n\n @Test\n def testConnectorEnv(): Unit = {\n val expected = if (StringUtils.isNotBlank(System.getenv(\"DATABRICKS_RUNTIME_VERSION\"))) {\n \"databricks\"\n } else {\n \"spark\"\n }\n val actual = Neo4jUtil.connectorEnv\n Assert.assertEquals(expected, actual)\n }\n\n @Test\n def testConnectorEnvForCustom(): Unit = {\n System.setProperty(\"neo4j.spark.platform\", \"abc\")\n val actual = Neo4jUtil.connectorEnv\n Assert.assertEquals(\"abc\", actual)\n }\n\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/util/ValidationsIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport org.hamcrest.CoreMatchers\nimport org.junit.Rule\nimport org.junit.Test\nimport org.junit.rules.ExpectedException\nimport org.neo4j.driver.AccessMode\nimport org.neo4j.spark.SparkConnectorScalaSuiteIT\nimport org.neo4j.spark.SparkConnectorScalaSuiteIT.neo4j\nimport org.neo4j.spark.TestUtil\n\nimport java.util.regex.Pattern\n\nimport scala.annotation.meta.getter\n\nclass ValidationsIT extends SparkConnectorScalaSuiteIT {\n\n @(Rule @getter)\n val expectedException: ExpectedException = ExpectedException.none\n\n @Test\n def testReadQueryShouldBeSyntacticallyInvalid(): Unit = {\n // then\n expectedException.expect(classOf[IllegalArgumentException])\n expectedException.expectMessage(\n CoreMatchers.containsString(\"Query not compiled for the following exception: ClientException: Invalid input \")\n )\n val query = \"MATCH (f{) RETURN f\"\n expectedException.expectMessage(CoreMatchers.containsString(query))\n\n // given\n val readOpts: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n readOpts.put(Neo4jOptions.URL, SparkConnectorScalaSuiteIT.server.getBoltUrl)\n readOpts.put(\"query\", query)\n\n // when\n Validations.validate(ValidateRead(neo4j, new Neo4jOptions(readOpts), \"1\"))\n }\n\n @Test\n def testReadQueryShouldBeSemanticallyInvalid(): Unit = {\n // then\n val query = \"MERGE (n:TestNode{id: 1}) RETURN n\"\n expectedException.expect(classOf[IllegalArgumentException])\n expectedException.expectMessage(\n s\"Invalid query `$query` because the accepted types are [READ_ONLY], but the actual type is READ_WRITE\"\n )\n\n // given\n val readOpts: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n readOpts.put(Neo4jOptions.URL, SparkConnectorScalaSuiteIT.server.getBoltUrl)\n readOpts.put(\"query\", query)\n\n // when\n Validations.validate(ValidateRead(neo4j, new Neo4jOptions(readOpts), \"1\"))\n }\n\n @Test\n def testReadQueryCountBeSyntacticallyInvalid(): Unit = {\n // then\n val query = \"MATCH (f{) RETURN f\"\n expectedException.expect(classOf[IllegalArgumentException])\n expectedException.expectMessage(CoreMatchers.containsString(\n \"Query count not compiled for the following exception: ClientException: Invalid input \"\n ))\n expectedException.expectMessage(CoreMatchers.containsString(s\"EXPLAIN $query\"))\n\n // given\n val readOpts: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n readOpts.put(Neo4jOptions.URL, SparkConnectorScalaSuiteIT.server.getBoltUrl)\n readOpts.put(\"query\", \"MATCH (f) RETURN f\")\n readOpts.put(\"query.count\", query)\n\n // when\n Validations.validate(ValidateRead(neo4j, new Neo4jOptions(readOpts), \"1\"))\n }\n\n @Test\n def testScriptQueryCountShouldContainAnInvalidQuery(): Unit = {\n // then\n expectedException.expect(classOf[IllegalArgumentException])\n expectedException.expectMessage(\n CoreMatchers.containsString(\"The following queries inside the `script` are not valid,\")\n )\n expectedException.expectMessage(\n CoreMatchers.containsString(\"Query not compiled for the following exception: ClientException: Invalid input \")\n )\n expectedException.expectMessage(CoreMatchers.containsString(\"EXPLAIN RETUR 2 AS two\"))\n\n // given\n val readOpts: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n readOpts.put(Neo4jOptions.URL, SparkConnectorScalaSuiteIT.server.getBoltUrl)\n readOpts.put(\"query\", \"MATCH (f) RETURN f\")\n readOpts.put(\"script\", \"RETURN 1 AS one; RETUR 2 AS two; RETURN 3 AS three\")\n\n // when\n Validations.validate(ValidateRead(neo4j, new Neo4jOptions(readOpts), \"1\"))\n }\n\n @Test\n def testWriteQueryShouldBeSyntacticallyInvalid(): Unit = {\n // then\n val query = \"MERGE (f{) RETURN f\"\n expectedException.expect(classOf[IllegalArgumentException])\n expectedException.expectMessage(\n CoreMatchers.containsString(\"Query not compiled for the following exception: ClientException: Invalid input \")\n )\n expectedException.expectMessage(CoreMatchers.containsString(query))\n\n // given\n val writeOpts: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n writeOpts.put(Neo4jOptions.URL, SparkConnectorScalaSuiteIT.server.getBoltUrl)\n writeOpts.put(Neo4jOptions.ACCESS_MODE, AccessMode.WRITE.toString)\n writeOpts.put(\"query\", query)\n\n // when\n Validations.validate(ValidateWrite(neo4j, new Neo4jOptions(writeOpts), \"1\", null))\n }\n\n @Test\n def testWriteQueryShouldBeSemanticallyInvalid(): Unit = {\n // then\n val query = \"MATCH (n:TestNode{id: 1}) RETURN n\"\n expectedException.expect(classOf[IllegalArgumentException])\n expectedException.expectMessage(\n s\"Invalid query `$query` because the accepted types are [WRITE_ONLY, READ_WRITE], but the actual type is READ_ONLY\"\n )\n\n // given\n val writeOpts: java.util.Map[String, String] = new java.util.HashMap[String, String]()\n writeOpts.put(Neo4jOptions.URL, SparkConnectorScalaSuiteIT.server.getBoltUrl)\n writeOpts.put(Neo4jOptions.ACCESS_MODE, AccessMode.WRITE.toString)\n writeOpts.put(\"query\", query)\n\n // when\n Validations.validate(ValidateWrite(neo4j, new Neo4jOptions(writeOpts), \"1\", null))\n }\n\n}\n" }, { "path": "common/src/test/scala/org/neo4j/spark/util/ValidationsTest.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.util\n\nimport org.apache.spark.sql.SparkSession\nimport org.junit\nimport org.junit.Assert.assertEquals\nimport org.junit.Test\nimport org.neo4j.spark.SparkConnectorScalaBaseTSE\n\nclass ValidationsTest extends SparkConnectorScalaBaseTSE {\n\n @Test\n def testVersionThrowsExceptionSparkVersionIsNotSupported(): Unit = {\n val sparkVersion = SparkSession.getActiveSession\n .map { _.version }\n .getOrElse(\"UNKNOWN\")\n try {\n Validations.validate(ValidateSparkMinVersion(\"3.10000\"))\n fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n } catch {\n case e: IllegalArgumentException =>\n assertEquals(\n s\"\"\"Your current Spark version $sparkVersion is not supported by the current connector.\n |Please visit https://neo4j.com/developer/spark/overview/#_spark_compatibility to know which connector version you need.\n |\"\"\".stripMargin,\n e.getMessage\n )\n case e: Throwable =>\n fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}, got ${e.getClass} instead\")\n }\n }\n\n @Test\n def testVersionShouldBeValid(): Unit = {\n val fullVersion = SparkSession\n .getDefaultSession\n .map(_.version)\n .getOrElse(\"3.2\")\n val baseVersion = fullVersion\n .split(\"\\\\.\")\n .take(2)\n .mkString(\".\")\n Validations.validate(ValidateSparkMinVersion(s\"$baseVersion.*\"))\n Validations.validate(ValidateSparkMinVersion(fullVersion))\n Validations.validate(ValidateSparkMinVersion(s\"$fullVersion-amzn-0\"))\n }\n\n @Test\n def testVersionShouldValidateTheVersion(): Unit = {\n val version = ValidateSparkMinVersion(\"2.3.0\")\n junit.Assert.assertTrue(version.isSupported(\"2.3.0-amzn-1\"))\n junit.Assert.assertTrue(version.isSupported(\"2.3.1-amzn-1\"))\n junit.Assert.assertTrue(version.isSupported(\"3.3.0-amzn-1\"))\n junit.Assert.assertTrue(version.isSupported(\"3.3.0\"))\n junit.Assert.assertTrue(version.isSupported(\"3.1.0\"))\n junit.Assert.assertTrue(version.isSupported(\"3.2.0\"))\n junit.Assert.assertFalse(version.isSupported(\"2.2.10\"))\n }\n\n}\n" }, { "path": "dangerfile.mjs", "content": "import load from '@commitlint/load';\nimport lint from '@commitlint/lint';\n\nconst minPRDescriptionLength = 10;\n\n// Utility functions\nconst processReport = (type, report, warnOnly = false) => {\n if (report.warnings.length > 0) {\n warn(\n `${type} '${report.input}': ${report.warnings\n .map((w) => w.message)\n .join(', ')}`,\n );\n }\n\n if (report.errors.length > 0) {\n const reportFn = warnOnly ? warn : fail;\n\n reportFn(\n `${type} '${report.input}': ${report.errors\n .map((e) => e.message)\n .join(', ')}`,\n );\n }\n\n return report.valid || warnOnly ? Promise.resolve() : Promise.reject();\n};\n\nconst reportCommitMessage = (report) => processReport('Commit Message', report, true);\n\nconst reportPRTitle = (report) => processReport('PR Title', report, false);\n\nconst lintMessage = (message, opts, reporter) =>\n lint(\n message,\n opts.rules,\n opts.parserPreset ? {parserOpts: opts.parserPreset.parserOpts} : {},\n ).then(reporter);\n\nconst pr = danger.github.pr;\n\n// check commit messages and PR name\nschedule(\n Promise.all([\n load({}, {file: './.commitlintrc.json', cwd: process.cwd()}).then(\n (opts) =>\n Promise.all(\n danger.git.commits\n .map((c) => c.message)\n .map((m) => lintMessage(m, opts, reportCommitMessage)),\n )\n .catch(() =>\n markdown(\n '> All commits should follow ' +\n '[Conventional commits](https://cheatography.com/albelop/cheat-sheets/conventional-commits). ' +\n 'It seems some of the commit messages are not following those rules, please fix them.',\n ),\n )\n .then(() => lintMessage(pr.title, opts, reportPRTitle))\n .catch(() =>\n markdown(\n '> Pull request title should follow ' +\n '[Conventional commits](https://cheatography.com/albelop/cheat-sheets/conventional-commits).',\n ),\n ),\n ),\n new Promise((resolve, reject) => {\n // No PR is too small to include a description of why you made a change\n if (pr.body.length < minPRDescriptionLength) {\n warn(`:exclamation: Please include a description of your PR changes.`);\n\n markdown(\n '> Pull request should have a description of the underlying changes.',\n );\n }\n }),\n ]),\n);\n" }, { "path": "examples/neo4j_data_engineering.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Open this notebook in Google Colab \\n\",\n \" \\\"Open\\n\",\n \"\"\n ],\n \"metadata\": {\n \"id\": \"EhTThKJMxDCy\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"7Nvb-_bYx359\"\n },\n \"source\": [\n \"# Example of a Simple data engineering workflow with Neo4j and Spark\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"This notebook contains a set of examples that explains how to extract insights from data using the Neo4j Connector for Apache Spark in a Data Engineering workflow with [AuraDB](https://neo4j.com/docs/aura/auradb/) our fully managed version of Neo4j database.\\n\",\n \"\\n\",\n \"The notebooks will enable you to test your knowledge with a set of exercises after each section.\\n\",\n \"\\n\",\n \"If you have any questions or problems feel free to write a post in the [Neo4j community forum](https://community.neo4j.com/) or in [Discord](https://discord.com/invite/neo4j).\\n\",\n \"\\n\",\n \"If you want more exercises feel free to open an issue in the [GitHub repository](https://github.com/neo4j/neo4j-spark-connector).\\n\",\n \"\\n\",\n \"Enjoy!\"\n ],\n \"metadata\": {\n \"id\": \"e0bo6ido8tL7\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"hXwkjQMnMXED\"\n },\n \"source\": [\n \"### Configure the Spark Environment\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"BhZwh-RAz6Bo\"\n },\n \"outputs\": [],\n \"source\": [\n \"!apt-get install openjdk-17-jdk-headless -qq > /dev/null\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"spark_version = '3.3.4'\"\n ],\n \"metadata\": {\n \"id\": \"gmEzhrux7Jek\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"!wget -q https://dlcdn.apache.org/spark/spark-$spark_version/spark-$spark_version-bin-hadoop3.tgz\"\n ],\n \"metadata\": {\n \"id\": \"Ya6Nj_u3vdTL\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"A3gsnSHl0F99\"\n },\n \"outputs\": [],\n \"source\": [\n \"!tar xf spark-$spark_version-bin-hadoop3.tgz\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"hSBQWKs90vSx\"\n },\n \"outputs\": [],\n \"source\": [\n \"!pip install -q findspark\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"tnW0a1Gj080k\"\n },\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"os.environ[\\\"JAVA_HOME\\\"] = \\\"/usr/lib/jvm/java-17-openjdk-amd64\\\"\\n\",\n \"os.environ[\\\"SPARK_HOME\\\"] = f\\\"/content/spark-{spark_version}-bin-hadoop3\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"dlUBSezK1DpZ\"\n },\n \"outputs\": [],\n \"source\": [\n \"import findspark\\n\",\n \"findspark.init()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"rd5KWGQiOVDV\"\n },\n \"outputs\": [],\n \"source\": [\n \"neo4j_url = \\\"\\\" # put your neo4j url here\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"uXbi_82KOTzU\"\n },\n \"outputs\": [],\n \"source\": [\n \"neo4j_user = \\\"neo4j\\\" # put your neo4j user here\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"Sw50wjxxOUqt\"\n },\n \"outputs\": [],\n \"source\": [\n \"neo4j_password = \\\"\\\" # put your neo4j password here\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"dOUJ-W871Tur\"\n },\n \"outputs\": [],\n \"source\": [\n \"from pyspark.sql import SparkSession\\n\",\n \"spark = (SparkSession.builder\\n\",\n \" .master('local[*]')\\n\",\n \" .appName('Data engineering workflow with Neo4j and Spark')\\n\",\n \" .config('spark.ui.port', '4050')\\n\",\n \" # Just to show dataframes as tables\\n\",\n \" .config('spark.sql.repl.eagerEval.enabled', True)\\n\",\n \" .config('spark.jars.packages', 'org.neo4j:neo4j-connector-apache-spark_2.12:5.1.0_for_spark_3')\\n\",\n \" # As we're using always the same database instance we'll\\n\",\n \" # define them as global variables\\n\",\n \" # so we don't need to repeat them each time\\n\",\n \" .config(\\\"neo4j.url\\\", neo4j_url)\\n\",\n \" .config(\\\"neo4j.authentication.type\\\", \\\"basic\\\")\\n\",\n \" .config(\\\"neo4j.authentication.basic.username\\\", neo4j_user)\\n\",\n \" .config(\\\"neo4j.authentication.basic.password\\\", neo4j_password)\\n\",\n \" .getOrCreate())\\n\",\n \"spark\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"\\n\",\n \"## Exercises prerequisite\\n\",\n \"\\n\",\n \"In this notebook we and going to test your knowledge. Some of the exercises require the Neo4j Python driver to check if the exercises are being solved correctly.\\n\",\n \"\\n\",\n \"*Neo4j Python Driver is required only for verifying the exercises when you persist data from Spark to Neo4j*\\n\",\n \"\\n\",\n \"**It's not required by the Spark connector!!!**\\n\",\n \"\\n\",\n \"We'll use [Cy2Py](https://github.com/conker84/cy2py), a Jupyter extension that easily allows you to connect to Neo4j and visualize data from Jupyter notebooks.\\n\",\n \"For a detailed instruction about how to use it please dive into [this example](https://github.com/conker84/cy2py/blob/main/examples/Neo4j_Crime_Investigation_Dataset.ipynb)\"\n ],\n \"metadata\": {\n \"id\": \"b6_YNZnZ5GdT\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"!pip install -q cy2py\"\n ],\n \"metadata\": {\n \"id\": \"f5ZZJylo5Bbz\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"CsnO4C9X7vK0\"\n },\n \"source\": [\n \"### Configure an Aura instance\\n\",\n \"\\n\",\n \"
\\n\",\n \"

Neo4j Aura DB is a fully managed cloud service: The zero-admin, always-on graph database for cloud developers.

\\n\",\n \"\\n\",\n \"Create a [free instance](https://console.neo4j.io/?ref=aura-lp&mpp=4bfb2414ab973c741b6f067bf06d5575&mpid=17f40ce03ac883-0f09bb214466c1-37677109-1ea000-17f40ce03ad975&_gl=1*ql4f6s*_ga*MTc2OTMwNjEwMy4xNjQ5NDI3MDE0*_ga_DL38Q8KGQC*MTY1MzQxMDQzMC43OS4xLjE2NTM0MTA3MjQuMA..&_ga=2.136543024.1659283742.1653295079-1769306103.1649427014&_gac=1.216269284.1653306922.CjwKCAjw4ayUBhA4EiwATWyBrl6dN0oaH9_btCfvzdhi77ieNP07GAkOYuz7wx9QEewBnG_FUIMg8xoCgLsQAvD_BwE)\\n\",\n \"\\n\",\n \"
\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"let's load the extension\"\n ],\n \"metadata\": {\n \"id\": \"uKYEPEgOcG2b\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%load_ext cy2py\"\n ],\n \"metadata\": {\n \"id\": \"38EeXF6icKOK\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Populate the database\\n\",\n \"\\n\",\n \"To perform this section go in the Neo4j Brower of your aura instance and paste the following query:\\n\",\n \"\\n\",\n \"
\\n\",\n \"\\n\",\n \"Show the Cypher query\\n\",\n \"\\n\",\n \"\\n\",\n \"```cypher\\n\",\n \"CREATE (TheMatrix:Movie {title:'The Matrix', released:1999, tagline:'Welcome to the Real World'})\\n\",\n \"CREATE (Keanu:Person {name:'Keanu Reeves', born:1964})\\n\",\n \"CREATE (Carrie:Person {name:'Carrie-Anne Moss', born:1967})\\n\",\n \"CREATE (Laurence:Person {name:'Laurence Fishburne', born:1961})\\n\",\n \"CREATE (Hugo:Person {name:'Hugo Weaving', born:1960})\\n\",\n \"CREATE (LillyW:Person {name:'Lilly Wachowski', born:1967})\\n\",\n \"CREATE (LanaW:Person {name:'Lana Wachowski', born:1965})\\n\",\n \"CREATE (JoelS:Person {name:'Joel Silver', born:1952})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Neo']}]->(TheMatrix),\\n\",\n \"(Carrie)-[:ACTED_IN {roles:['Trinity']}]->(TheMatrix),\\n\",\n \"(Laurence)-[:ACTED_IN {roles:['Morpheus']}]->(TheMatrix),\\n\",\n \"(Hugo)-[:ACTED_IN {roles:['Agent Smith']}]->(TheMatrix),\\n\",\n \"(LillyW)-[:DIRECTED]->(TheMatrix),\\n\",\n \"(LanaW)-[:DIRECTED]->(TheMatrix),\\n\",\n \"(JoelS)-[:PRODUCED]->(TheMatrix)\\n\",\n \"\\n\",\n \"CREATE (Emil:Person {name:\\\"Emil Eifrem\\\", born:1978})\\n\",\n \"CREATE (Emil)-[:ACTED_IN {roles:[\\\"Emil\\\"]}]->(TheMatrix)\\n\",\n \"\\n\",\n \"CREATE (TheMatrixReloaded:Movie {title:'The Matrix Reloaded', released:2003, tagline:'Free your mind'})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Neo']}]->(TheMatrixReloaded),\\n\",\n \"(Carrie)-[:ACTED_IN {roles:['Trinity']}]->(TheMatrixReloaded),\\n\",\n \"(Laurence)-[:ACTED_IN {roles:['Morpheus']}]->(TheMatrixReloaded),\\n\",\n \"(Hugo)-[:ACTED_IN {roles:['Agent Smith']}]->(TheMatrixReloaded),\\n\",\n \"(LillyW)-[:DIRECTED]->(TheMatrixReloaded),\\n\",\n \"(LanaW)-[:DIRECTED]->(TheMatrixReloaded),\\n\",\n \"(JoelS)-[:PRODUCED]->(TheMatrixReloaded)\\n\",\n \"\\n\",\n \"CREATE (TheMatrixRevolutions:Movie {title:'The Matrix Revolutions', released:2003, tagline:'Everything that has a beginning has an end'})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Neo']}]->(TheMatrixRevolutions),\\n\",\n \"(Carrie)-[:ACTED_IN {roles:['Trinity']}]->(TheMatrixRevolutions),\\n\",\n \"(Laurence)-[:ACTED_IN {roles:['Morpheus']}]->(TheMatrixRevolutions),\\n\",\n \"(Hugo)-[:ACTED_IN {roles:['Agent Smith']}]->(TheMatrixRevolutions),\\n\",\n \"(LillyW)-[:DIRECTED]->(TheMatrixRevolutions),\\n\",\n \"(LanaW)-[:DIRECTED]->(TheMatrixRevolutions),\\n\",\n \"(JoelS)-[:PRODUCED]->(TheMatrixRevolutions)\\n\",\n \"\\n\",\n \"CREATE (TheDevilsAdvocate:Movie {title:\\\"The Devil's Advocate\\\", released:1997, tagline:'Evil has its winning ways'})\\n\",\n \"CREATE (Charlize:Person {name:'Charlize Theron', born:1975})\\n\",\n \"CREATE (Al:Person {name:'Al Pacino', born:1940})\\n\",\n \"CREATE (Taylor:Person {name:'Taylor Hackford', born:1944})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Kevin Lomax']}]->(TheDevilsAdvocate),\\n\",\n \"(Charlize)-[:ACTED_IN {roles:['Mary Ann Lomax']}]->(TheDevilsAdvocate),\\n\",\n \"(Al)-[:ACTED_IN {roles:['John Milton']}]->(TheDevilsAdvocate),\\n\",\n \"(Taylor)-[:DIRECTED]->(TheDevilsAdvocate)\\n\",\n \"\\n\",\n \"CREATE (AFewGoodMen:Movie {title:\\\"A Few Good Men\\\", released:1992, tagline:\\\"In the heart of the nation's capital, in a courthouse of the U.S. government, one man will stop at nothing to keep his honor, and one will stop at nothing to find the truth.\\\"})\\n\",\n \"CREATE (TomC:Person {name:'Tom Cruise', born:1962})\\n\",\n \"CREATE (JackN:Person {name:'Jack Nicholson', born:1937})\\n\",\n \"CREATE (DemiM:Person {name:'Demi Moore', born:1962})\\n\",\n \"CREATE (KevinB:Person {name:'Kevin Bacon', born:1958})\\n\",\n \"CREATE (KieferS:Person {name:'Kiefer Sutherland', born:1966})\\n\",\n \"CREATE (NoahW:Person {name:'Noah Wyle', born:1971})\\n\",\n \"CREATE (CubaG:Person {name:'Cuba Gooding Jr.', born:1968})\\n\",\n \"CREATE (KevinP:Person {name:'Kevin Pollak', born:1957})\\n\",\n \"CREATE (JTW:Person {name:'J.T. Walsh', born:1943})\\n\",\n \"CREATE (JamesM:Person {name:'James Marshall', born:1967})\\n\",\n \"CREATE (ChristopherG:Person {name:'Christopher Guest', born:1948})\\n\",\n \"CREATE (RobR:Person {name:'Rob Reiner', born:1947})\\n\",\n \"CREATE (AaronS:Person {name:'Aaron Sorkin', born:1961})\\n\",\n \"CREATE\\n\",\n \"(TomC)-[:ACTED_IN {roles:['Lt. Daniel Kaffee']}]->(AFewGoodMen),\\n\",\n \"(JackN)-[:ACTED_IN {roles:['Col. Nathan R. Jessup']}]->(AFewGoodMen),\\n\",\n \"(DemiM)-[:ACTED_IN {roles:['Lt. Cdr. JoAnne Galloway']}]->(AFewGoodMen),\\n\",\n \"(KevinB)-[:ACTED_IN {roles:['Capt. Jack Ross']}]->(AFewGoodMen),\\n\",\n \"(KieferS)-[:ACTED_IN {roles:['Lt. Jonathan Kendrick']}]->(AFewGoodMen),\\n\",\n \"(NoahW)-[:ACTED_IN {roles:['Cpl. Jeffrey Barnes']}]->(AFewGoodMen),\\n\",\n \"(CubaG)-[:ACTED_IN {roles:['Cpl. Carl Hammaker']}]->(AFewGoodMen),\\n\",\n \"(KevinP)-[:ACTED_IN {roles:['Lt. Sam Weinberg']}]->(AFewGoodMen),\\n\",\n \"(JTW)-[:ACTED_IN {roles:['Lt. Col. Matthew Andrew Markinson']}]->(AFewGoodMen),\\n\",\n \"(JamesM)-[:ACTED_IN {roles:['Pfc. Louden Downey']}]->(AFewGoodMen),\\n\",\n \"(ChristopherG)-[:ACTED_IN {roles:['Dr. Stone']}]->(AFewGoodMen),\\n\",\n \"(AaronS)-[:ACTED_IN {roles:['Man in Bar']}]->(AFewGoodMen),\\n\",\n \"(RobR)-[:DIRECTED]->(AFewGoodMen),\\n\",\n \"(AaronS)-[:WROTE]->(AFewGoodMen)\\n\",\n \"\\n\",\n \"CREATE (TopGun:Movie {title:\\\"Top Gun\\\", released:1986, tagline:'I feel the need, the need for speed.'})\\n\",\n \"CREATE (KellyM:Person {name:'Kelly McGillis', born:1957})\\n\",\n \"CREATE (ValK:Person {name:'Val Kilmer', born:1959})\\n\",\n \"CREATE (AnthonyE:Person {name:'Anthony Edwards', born:1962})\\n\",\n \"CREATE (TomS:Person {name:'Tom Skerritt', born:1933})\\n\",\n \"CREATE (MegR:Person {name:'Meg Ryan', born:1961})\\n\",\n \"CREATE (TonyS:Person {name:'Tony Scott', born:1944})\\n\",\n \"CREATE (JimC:Person {name:'Jim Cash', born:1941})\\n\",\n \"CREATE\\n\",\n \"(TomC)-[:ACTED_IN {roles:['Maverick']}]->(TopGun),\\n\",\n \"(KellyM)-[:ACTED_IN {roles:['Charlie']}]->(TopGun),\\n\",\n \"(ValK)-[:ACTED_IN {roles:['Iceman']}]->(TopGun),\\n\",\n \"(AnthonyE)-[:ACTED_IN {roles:['Goose']}]->(TopGun),\\n\",\n \"(TomS)-[:ACTED_IN {roles:['Viper']}]->(TopGun),\\n\",\n \"(MegR)-[:ACTED_IN {roles:['Carole']}]->(TopGun),\\n\",\n \"(TonyS)-[:DIRECTED]->(TopGun),\\n\",\n \"(JimC)-[:WROTE]->(TopGun)\\n\",\n \"\\n\",\n \"CREATE (JerryMaguire:Movie {title:'Jerry Maguire', released:2000, tagline:'The rest of his life begins now.'})\\n\",\n \"CREATE (ReneeZ:Person {name:'Renee Zellweger', born:1969})\\n\",\n \"CREATE (KellyP:Person {name:'Kelly Preston', born:1962})\\n\",\n \"CREATE (JerryO:Person {name:\\\"Jerry O'Connell\\\", born:1974})\\n\",\n \"CREATE (JayM:Person {name:'Jay Mohr', born:1970})\\n\",\n \"CREATE (BonnieH:Person {name:'Bonnie Hunt', born:1961})\\n\",\n \"CREATE (ReginaK:Person {name:'Regina King', born:1971})\\n\",\n \"CREATE (JonathanL:Person {name:'Jonathan Lipnicki', born:1996})\\n\",\n \"CREATE (CameronC:Person {name:'Cameron Crowe', born:1957})\\n\",\n \"CREATE\\n\",\n \"(TomC)-[:ACTED_IN {roles:['Jerry Maguire']}]->(JerryMaguire),\\n\",\n \"(CubaG)-[:ACTED_IN {roles:['Rod Tidwell']}]->(JerryMaguire),\\n\",\n \"(ReneeZ)-[:ACTED_IN {roles:['Dorothy Boyd']}]->(JerryMaguire),\\n\",\n \"(KellyP)-[:ACTED_IN {roles:['Avery Bishop']}]->(JerryMaguire),\\n\",\n \"(JerryO)-[:ACTED_IN {roles:['Frank Cushman']}]->(JerryMaguire),\\n\",\n \"(JayM)-[:ACTED_IN {roles:['Bob Sugar']}]->(JerryMaguire),\\n\",\n \"(BonnieH)-[:ACTED_IN {roles:['Laurel Boyd']}]->(JerryMaguire),\\n\",\n \"(ReginaK)-[:ACTED_IN {roles:['Marcee Tidwell']}]->(JerryMaguire),\\n\",\n \"(JonathanL)-[:ACTED_IN {roles:['Ray Boyd']}]->(JerryMaguire),\\n\",\n \"(CameronC)-[:DIRECTED]->(JerryMaguire),\\n\",\n \"(CameronC)-[:PRODUCED]->(JerryMaguire),\\n\",\n \"(CameronC)-[:WROTE]->(JerryMaguire)\\n\",\n \"\\n\",\n \"CREATE (StandByMe:Movie {title:\\\"Stand By Me\\\", released:1986, tagline:\\\"For some, it's the last real taste of innocence, and the first real taste of life. But for everyone, it's the time that memories are made of.\\\"})\\n\",\n \"CREATE (RiverP:Person {name:'River Phoenix', born:1970})\\n\",\n \"CREATE (CoreyF:Person {name:'Corey Feldman', born:1971})\\n\",\n \"CREATE (WilW:Person {name:'Wil Wheaton', born:1972})\\n\",\n \"CREATE (JohnC:Person {name:'John Cusack', born:1966})\\n\",\n \"CREATE (MarshallB:Person {name:'Marshall Bell', born:1942})\\n\",\n \"CREATE\\n\",\n \"(WilW)-[:ACTED_IN {roles:['Gordie Lachance']}]->(StandByMe),\\n\",\n \"(RiverP)-[:ACTED_IN {roles:['Chris Chambers']}]->(StandByMe),\\n\",\n \"(JerryO)-[:ACTED_IN {roles:['Vern Tessio']}]->(StandByMe),\\n\",\n \"(CoreyF)-[:ACTED_IN {roles:['Teddy Duchamp']}]->(StandByMe),\\n\",\n \"(JohnC)-[:ACTED_IN {roles:['Denny Lachance']}]->(StandByMe),\\n\",\n \"(KieferS)-[:ACTED_IN {roles:['Ace Merrill']}]->(StandByMe),\\n\",\n \"(MarshallB)-[:ACTED_IN {roles:['Mr. Lachance']}]->(StandByMe),\\n\",\n \"(RobR)-[:DIRECTED]->(StandByMe)\\n\",\n \"\\n\",\n \"CREATE (AsGoodAsItGets:Movie {title:'As Good as It Gets', released:1997, tagline:'A comedy from the heart that goes for the throat.'})\\n\",\n \"CREATE (HelenH:Person {name:'Helen Hunt', born:1963})\\n\",\n \"CREATE (GregK:Person {name:'Greg Kinnear', born:1963})\\n\",\n \"CREATE (JamesB:Person {name:'James L. Brooks', born:1940})\\n\",\n \"CREATE\\n\",\n \"(JackN)-[:ACTED_IN {roles:['Melvin Udall']}]->(AsGoodAsItGets),\\n\",\n \"(HelenH)-[:ACTED_IN {roles:['Carol Connelly']}]->(AsGoodAsItGets),\\n\",\n \"(GregK)-[:ACTED_IN {roles:['Simon Bishop']}]->(AsGoodAsItGets),\\n\",\n \"(CubaG)-[:ACTED_IN {roles:['Frank Sachs']}]->(AsGoodAsItGets),\\n\",\n \"(JamesB)-[:DIRECTED]->(AsGoodAsItGets)\\n\",\n \"\\n\",\n \"CREATE (WhatDreamsMayCome:Movie {title:'What Dreams May Come', released:1998, tagline:'After life there is more. The end is just the beginning.'})\\n\",\n \"CREATE (AnnabellaS:Person {name:'Annabella Sciorra', born:1960})\\n\",\n \"CREATE (MaxS:Person {name:'Max von Sydow', born:1929})\\n\",\n \"CREATE (WernerH:Person {name:'Werner Herzog', born:1942})\\n\",\n \"CREATE (Robin:Person {name:'Robin Williams', born:1951})\\n\",\n \"CREATE (VincentW:Person {name:'Vincent Ward', born:1956})\\n\",\n \"CREATE\\n\",\n \"(Robin)-[:ACTED_IN {roles:['Chris Nielsen']}]->(WhatDreamsMayCome),\\n\",\n \"(CubaG)-[:ACTED_IN {roles:['Albert Lewis']}]->(WhatDreamsMayCome),\\n\",\n \"(AnnabellaS)-[:ACTED_IN {roles:['Annie Collins-Nielsen']}]->(WhatDreamsMayCome),\\n\",\n \"(MaxS)-[:ACTED_IN {roles:['The Tracker']}]->(WhatDreamsMayCome),\\n\",\n \"(WernerH)-[:ACTED_IN {roles:['The Face']}]->(WhatDreamsMayCome),\\n\",\n \"(VincentW)-[:DIRECTED]->(WhatDreamsMayCome)\\n\",\n \"\\n\",\n \"CREATE (SnowFallingonCedars:Movie {title:'Snow Falling on Cedars', released:1999, tagline:'First loves last. Forever.'})\\n\",\n \"CREATE (EthanH:Person {name:'Ethan Hawke', born:1970})\\n\",\n \"CREATE (RickY:Person {name:'Rick Yune', born:1971})\\n\",\n \"CREATE (JamesC:Person {name:'James Cromwell', born:1940})\\n\",\n \"CREATE (ScottH:Person {name:'Scott Hicks', born:1953})\\n\",\n \"CREATE\\n\",\n \"(EthanH)-[:ACTED_IN {roles:['Ishmael Chambers']}]->(SnowFallingonCedars),\\n\",\n \"(RickY)-[:ACTED_IN {roles:['Kazuo Miyamoto']}]->(SnowFallingonCedars),\\n\",\n \"(MaxS)-[:ACTED_IN {roles:['Nels Gudmundsson']}]->(SnowFallingonCedars),\\n\",\n \"(JamesC)-[:ACTED_IN {roles:['Judge Fielding']}]->(SnowFallingonCedars),\\n\",\n \"(ScottH)-[:DIRECTED]->(SnowFallingonCedars)\\n\",\n \"\\n\",\n \"CREATE (YouveGotMail:Movie {title:\\\"You've Got Mail\\\", released:1998, tagline:'At odds in life... in love on-line.'})\\n\",\n \"CREATE (ParkerP:Person {name:'Parker Posey', born:1968})\\n\",\n \"CREATE (DaveC:Person {name:'Dave Chappelle', born:1973})\\n\",\n \"CREATE (SteveZ:Person {name:'Steve Zahn', born:1967})\\n\",\n \"CREATE (TomH:Person {name:'Tom Hanks', born:1956})\\n\",\n \"CREATE (NoraE:Person {name:'Nora Ephron', born:1941})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Joe Fox']}]->(YouveGotMail),\\n\",\n \"(MegR)-[:ACTED_IN {roles:['Kathleen Kelly']}]->(YouveGotMail),\\n\",\n \"(GregK)-[:ACTED_IN {roles:['Frank Navasky']}]->(YouveGotMail),\\n\",\n \"(ParkerP)-[:ACTED_IN {roles:['Patricia Eden']}]->(YouveGotMail),\\n\",\n \"(DaveC)-[:ACTED_IN {roles:['Kevin Jackson']}]->(YouveGotMail),\\n\",\n \"(SteveZ)-[:ACTED_IN {roles:['George Pappas']}]->(YouveGotMail),\\n\",\n \"(NoraE)-[:DIRECTED]->(YouveGotMail)\\n\",\n \"\\n\",\n \"CREATE (SleeplessInSeattle:Movie {title:'Sleepless in Seattle', released:1993, tagline:'What if someone you never met, someone you never saw, someone you never knew was the only someone for you?'})\\n\",\n \"CREATE (RitaW:Person {name:'Rita Wilson', born:1956})\\n\",\n \"CREATE (BillPull:Person {name:'Bill Pullman', born:1953})\\n\",\n \"CREATE (VictorG:Person {name:'Victor Garber', born:1949})\\n\",\n \"CREATE (RosieO:Person {name:\\\"Rosie O'Donnell\\\", born:1962})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Sam Baldwin']}]->(SleeplessInSeattle),\\n\",\n \"(MegR)-[:ACTED_IN {roles:['Annie Reed']}]->(SleeplessInSeattle),\\n\",\n \"(RitaW)-[:ACTED_IN {roles:['Suzy']}]->(SleeplessInSeattle),\\n\",\n \"(BillPull)-[:ACTED_IN {roles:['Walter']}]->(SleeplessInSeattle),\\n\",\n \"(VictorG)-[:ACTED_IN {roles:['Greg']}]->(SleeplessInSeattle),\\n\",\n \"(RosieO)-[:ACTED_IN {roles:['Becky']}]->(SleeplessInSeattle),\\n\",\n \"(NoraE)-[:DIRECTED]->(SleeplessInSeattle)\\n\",\n \"\\n\",\n \"CREATE (JoeVersustheVolcano:Movie {title:'Joe Versus the Volcano', released:1990, tagline:'A story of love, lava and burning desire.'})\\n\",\n \"CREATE (JohnS:Person {name:'John Patrick Stanley', born:1950})\\n\",\n \"CREATE (Nathan:Person {name:'Nathan Lane', born:1956})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Joe Banks']}]->(JoeVersustheVolcano),\\n\",\n \"(MegR)-[:ACTED_IN {roles:['DeDe', 'Angelica Graynamore', 'Patricia Graynamore']}]->(JoeVersustheVolcano),\\n\",\n \"(Nathan)-[:ACTED_IN {roles:['Baw']}]->(JoeVersustheVolcano),\\n\",\n \"(JohnS)-[:DIRECTED]->(JoeVersustheVolcano)\\n\",\n \"\\n\",\n \"CREATE (WhenHarryMetSally:Movie {title:'When Harry Met Sally', released:1998, tagline:'Can two friends sleep together and still love each other in the morning?'})\\n\",\n \"CREATE (BillyC:Person {name:'Billy Crystal', born:1948})\\n\",\n \"CREATE (CarrieF:Person {name:'Carrie Fisher', born:1956})\\n\",\n \"CREATE (BrunoK:Person {name:'Bruno Kirby', born:1949})\\n\",\n \"CREATE\\n\",\n \"(BillyC)-[:ACTED_IN {roles:['Harry Burns']}]->(WhenHarryMetSally),\\n\",\n \"(MegR)-[:ACTED_IN {roles:['Sally Albright']}]->(WhenHarryMetSally),\\n\",\n \"(CarrieF)-[:ACTED_IN {roles:['Marie']}]->(WhenHarryMetSally),\\n\",\n \"(BrunoK)-[:ACTED_IN {roles:['Jess']}]->(WhenHarryMetSally),\\n\",\n \"(RobR)-[:DIRECTED]->(WhenHarryMetSally),\\n\",\n \"(RobR)-[:PRODUCED]->(WhenHarryMetSally),\\n\",\n \"(NoraE)-[:PRODUCED]->(WhenHarryMetSally),\\n\",\n \"(NoraE)-[:WROTE]->(WhenHarryMetSally)\\n\",\n \"\\n\",\n \"CREATE (ThatThingYouDo:Movie {title:'That Thing You Do', released:1996, tagline:'In every life there comes a time when that thing you dream becomes that thing you do'})\\n\",\n \"CREATE (LivT:Person {name:'Liv Tyler', born:1977})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Mr. White']}]->(ThatThingYouDo),\\n\",\n \"(LivT)-[:ACTED_IN {roles:['Faye Dolan']}]->(ThatThingYouDo),\\n\",\n \"(Charlize)-[:ACTED_IN {roles:['Tina']}]->(ThatThingYouDo),\\n\",\n \"(TomH)-[:DIRECTED]->(ThatThingYouDo)\\n\",\n \"\\n\",\n \"CREATE (TheReplacements:Movie {title:'The Replacements', released:2000, tagline:'Pain heals, Chicks dig scars... Glory lasts forever'})\\n\",\n \"CREATE (Brooke:Person {name:'Brooke Langton', born:1970})\\n\",\n \"CREATE (Gene:Person {name:'Gene Hackman', born:1930})\\n\",\n \"CREATE (Orlando:Person {name:'Orlando Jones', born:1968})\\n\",\n \"CREATE (Howard:Person {name:'Howard Deutch', born:1950})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Shane Falco']}]->(TheReplacements),\\n\",\n \"(Brooke)-[:ACTED_IN {roles:['Annabelle Farrell']}]->(TheReplacements),\\n\",\n \"(Gene)-[:ACTED_IN {roles:['Jimmy McGinty']}]->(TheReplacements),\\n\",\n \"(Orlando)-[:ACTED_IN {roles:['Clifford Franklin']}]->(TheReplacements),\\n\",\n \"(Howard)-[:DIRECTED]->(TheReplacements)\\n\",\n \"\\n\",\n \"CREATE (RescueDawn:Movie {title:'RescueDawn', released:2006, tagline:\\\"Based on the extraordinary true story of one man's fight for freedom\\\"})\\n\",\n \"CREATE (ChristianB:Person {name:'Christian Bale', born:1974})\\n\",\n \"CREATE (ZachG:Person {name:'Zach Grenier', born:1954})\\n\",\n \"CREATE\\n\",\n \"(MarshallB)-[:ACTED_IN {roles:['Admiral']}]->(RescueDawn),\\n\",\n \"(ChristianB)-[:ACTED_IN {roles:['Dieter Dengler']}]->(RescueDawn),\\n\",\n \"(ZachG)-[:ACTED_IN {roles:['Squad Leader']}]->(RescueDawn),\\n\",\n \"(SteveZ)-[:ACTED_IN {roles:['Duane']}]->(RescueDawn),\\n\",\n \"(WernerH)-[:DIRECTED]->(RescueDawn)\\n\",\n \"\\n\",\n \"CREATE (TheBirdcage:Movie {title:'The Birdcage', released:1996, tagline:'Come as you are'})\\n\",\n \"CREATE (MikeN:Person {name:'Mike Nichols', born:1931})\\n\",\n \"CREATE\\n\",\n \"(Robin)-[:ACTED_IN {roles:['Armand Goldman']}]->(TheBirdcage),\\n\",\n \"(Nathan)-[:ACTED_IN {roles:['Albert Goldman']}]->(TheBirdcage),\\n\",\n \"(Gene)-[:ACTED_IN {roles:['Sen. Kevin Keeley']}]->(TheBirdcage),\\n\",\n \"(MikeN)-[:DIRECTED]->(TheBirdcage)\\n\",\n \"\\n\",\n \"CREATE (Unforgiven:Movie {title:'Unforgiven', released:1992, tagline:\\\"It's a hell of a thing, killing a man\\\"})\\n\",\n \"CREATE (RichardH:Person {name:'Richard Harris', born:1930})\\n\",\n \"CREATE (ClintE:Person {name:'Clint Eastwood', born:1930})\\n\",\n \"CREATE\\n\",\n \"(RichardH)-[:ACTED_IN {roles:['English Bob']}]->(Unforgiven),\\n\",\n \"(ClintE)-[:ACTED_IN {roles:['Bill Munny']}]->(Unforgiven),\\n\",\n \"(Gene)-[:ACTED_IN {roles:['Little Bill Daggett']}]->(Unforgiven),\\n\",\n \"(ClintE)-[:DIRECTED]->(Unforgiven)\\n\",\n \"\\n\",\n \"CREATE (JohnnyMnemonic:Movie {title:'Johnny Mnemonic', released:1995, tagline:'The hottest data on earth. In the coolest head in town'})\\n\",\n \"CREATE (Takeshi:Person {name:'Takeshi Kitano', born:1947})\\n\",\n \"CREATE (Dina:Person {name:'Dina Meyer', born:1968})\\n\",\n \"CREATE (IceT:Person {name:'Ice-T', born:1958})\\n\",\n \"CREATE (RobertL:Person {name:'Robert Longo', born:1953})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Johnny Mnemonic']}]->(JohnnyMnemonic),\\n\",\n \"(Takeshi)-[:ACTED_IN {roles:['Takahashi']}]->(JohnnyMnemonic),\\n\",\n \"(Dina)-[:ACTED_IN {roles:['Jane']}]->(JohnnyMnemonic),\\n\",\n \"(IceT)-[:ACTED_IN {roles:['J-Bone']}]->(JohnnyMnemonic),\\n\",\n \"(RobertL)-[:DIRECTED]->(JohnnyMnemonic)\\n\",\n \"\\n\",\n \"CREATE (CloudAtlas:Movie {title:'Cloud Atlas', released:2012, tagline:'Everything is connected'})\\n\",\n \"CREATE (HalleB:Person {name:'Halle Berry', born:1966})\\n\",\n \"CREATE (JimB:Person {name:'Jim Broadbent', born:1949})\\n\",\n \"CREATE (TomT:Person {name:'Tom Tykwer', born:1965})\\n\",\n \"CREATE (DavidMitchell:Person {name:'David Mitchell', born:1969})\\n\",\n \"CREATE (StefanArndt:Person {name:'Stefan Arndt', born:1961})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Zachry', 'Dr. Henry Goose', 'Isaac Sachs', 'Dermot Hoggins']}]->(CloudAtlas),\\n\",\n \"(Hugo)-[:ACTED_IN {roles:['Bill Smoke', 'Haskell Moore', 'Tadeusz Kesselring', 'Nurse Noakes', 'Boardman Mephi', 'Old Georgie']}]->(CloudAtlas),\\n\",\n \"(HalleB)-[:ACTED_IN {roles:['Luisa Rey', 'Jocasta Ayrs', 'Ovid', 'Meronym']}]->(CloudAtlas),\\n\",\n \"(JimB)-[:ACTED_IN {roles:['Vyvyan Ayrs', 'Captain Molyneux', 'Timothy Cavendish']}]->(CloudAtlas),\\n\",\n \"(TomT)-[:DIRECTED]->(CloudAtlas),\\n\",\n \"(LillyW)-[:DIRECTED]->(CloudAtlas),\\n\",\n \"(LanaW)-[:DIRECTED]->(CloudAtlas),\\n\",\n \"(DavidMitchell)-[:WROTE]->(CloudAtlas),\\n\",\n \"(StefanArndt)-[:PRODUCED]->(CloudAtlas)\\n\",\n \"\\n\",\n \"CREATE (TheDaVinciCode:Movie {title:'The Da Vinci Code', released:2006, tagline:'Break The Codes'})\\n\",\n \"CREATE (IanM:Person {name:'Ian McKellen', born:1939})\\n\",\n \"CREATE (AudreyT:Person {name:'Audrey Tautou', born:1976})\\n\",\n \"CREATE (PaulB:Person {name:'Paul Bettany', born:1971})\\n\",\n \"CREATE (RonH:Person {name:'Ron Howard', born:1954})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Dr. Robert Langdon']}]->(TheDaVinciCode),\\n\",\n \"(IanM)-[:ACTED_IN {roles:['Sir Leight Teabing']}]->(TheDaVinciCode),\\n\",\n \"(AudreyT)-[:ACTED_IN {roles:['Sophie Neveu']}]->(TheDaVinciCode),\\n\",\n \"(PaulB)-[:ACTED_IN {roles:['Silas']}]->(TheDaVinciCode),\\n\",\n \"(RonH)-[:DIRECTED]->(TheDaVinciCode)\\n\",\n \"\\n\",\n \"CREATE (VforVendetta:Movie {title:'V for Vendetta', released:2006, tagline:'Freedom! Forever!'})\\n\",\n \"CREATE (NatalieP:Person {name:'Natalie Portman', born:1981})\\n\",\n \"CREATE (StephenR:Person {name:'Stephen Rea', born:1946})\\n\",\n \"CREATE (JohnH:Person {name:'John Hurt', born:1940})\\n\",\n \"CREATE (BenM:Person {name: 'Ben Miles', born:1967})\\n\",\n \"CREATE\\n\",\n \"(Hugo)-[:ACTED_IN {roles:['V']}]->(VforVendetta),\\n\",\n \"(NatalieP)-[:ACTED_IN {roles:['Evey Hammond']}]->(VforVendetta),\\n\",\n \"(StephenR)-[:ACTED_IN {roles:['Eric Finch']}]->(VforVendetta),\\n\",\n \"(JohnH)-[:ACTED_IN {roles:['High Chancellor Adam Sutler']}]->(VforVendetta),\\n\",\n \"(BenM)-[:ACTED_IN {roles:['Dascomb']}]->(VforVendetta),\\n\",\n \"(JamesM)-[:DIRECTED]->(VforVendetta),\\n\",\n \"(LillyW)-[:PRODUCED]->(VforVendetta),\\n\",\n \"(LanaW)-[:PRODUCED]->(VforVendetta),\\n\",\n \"(JoelS)-[:PRODUCED]->(VforVendetta),\\n\",\n \"(LillyW)-[:WROTE]->(VforVendetta),\\n\",\n \"(LanaW)-[:WROTE]->(VforVendetta)\\n\",\n \"\\n\",\n \"CREATE (SpeedRacer:Movie {title:'Speed Racer', released:2008, tagline:'Speed has no limits'})\\n\",\n \"CREATE (EmileH:Person {name:'Emile Hirsch', born:1985})\\n\",\n \"CREATE (JohnG:Person {name:'John Goodman', born:1960})\\n\",\n \"CREATE (SusanS:Person {name:'Susan Sarandon', born:1946})\\n\",\n \"CREATE (MatthewF:Person {name:'Matthew Fox', born:1966})\\n\",\n \"CREATE (ChristinaR:Person {name:'Christina Ricci', born:1980})\\n\",\n \"CREATE (Rain:Person {name:'Rain', born:1982})\\n\",\n \"CREATE\\n\",\n \"(EmileH)-[:ACTED_IN {roles:['Speed Racer']}]->(SpeedRacer),\\n\",\n \"(JohnG)-[:ACTED_IN {roles:['Pops']}]->(SpeedRacer),\\n\",\n \"(SusanS)-[:ACTED_IN {roles:['Mom']}]->(SpeedRacer),\\n\",\n \"(MatthewF)-[:ACTED_IN {roles:['Racer X']}]->(SpeedRacer),\\n\",\n \"(ChristinaR)-[:ACTED_IN {roles:['Trixie']}]->(SpeedRacer),\\n\",\n \"(Rain)-[:ACTED_IN {roles:['Taejo Togokahn']}]->(SpeedRacer),\\n\",\n \"(BenM)-[:ACTED_IN {roles:['Cass Jones']}]->(SpeedRacer),\\n\",\n \"(LillyW)-[:DIRECTED]->(SpeedRacer),\\n\",\n \"(LanaW)-[:DIRECTED]->(SpeedRacer),\\n\",\n \"(LillyW)-[:WROTE]->(SpeedRacer),\\n\",\n \"(LanaW)-[:WROTE]->(SpeedRacer),\\n\",\n \"(JoelS)-[:PRODUCED]->(SpeedRacer)\\n\",\n \"\\n\",\n \"CREATE (NinjaAssassin:Movie {title:'Ninja Assassin', released:2009, tagline:'Prepare to enter a secret world of assassins'})\\n\",\n \"CREATE (NaomieH:Person {name:'Naomie Harris'})\\n\",\n \"CREATE\\n\",\n \"(Rain)-[:ACTED_IN {roles:['Raizo']}]->(NinjaAssassin),\\n\",\n \"(NaomieH)-[:ACTED_IN {roles:['Mika Coretti']}]->(NinjaAssassin),\\n\",\n \"(RickY)-[:ACTED_IN {roles:['Takeshi']}]->(NinjaAssassin),\\n\",\n \"(BenM)-[:ACTED_IN {roles:['Ryan Maslow']}]->(NinjaAssassin),\\n\",\n \"(JamesM)-[:DIRECTED]->(NinjaAssassin),\\n\",\n \"(LillyW)-[:PRODUCED]->(NinjaAssassin),\\n\",\n \"(LanaW)-[:PRODUCED]->(NinjaAssassin),\\n\",\n \"(JoelS)-[:PRODUCED]->(NinjaAssassin)\\n\",\n \"\\n\",\n \"CREATE (TheGreenMile:Movie {title:'The Green Mile', released:1999, tagline:\\\"Walk a mile you'll never forget.\\\"})\\n\",\n \"CREATE (MichaelD:Person {name:'Michael Clarke Duncan', born:1957})\\n\",\n \"CREATE (DavidM:Person {name:'David Morse', born:1953})\\n\",\n \"CREATE (SamR:Person {name:'Sam Rockwell', born:1968})\\n\",\n \"CREATE (GaryS:Person {name:'Gary Sinise', born:1955})\\n\",\n \"CREATE (PatriciaC:Person {name:'Patricia Clarkson', born:1959})\\n\",\n \"CREATE (FrankD:Person {name:'Frank Darabont', born:1959})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Paul Edgecomb']}]->(TheGreenMile),\\n\",\n \"(MichaelD)-[:ACTED_IN {roles:['John Coffey']}]->(TheGreenMile),\\n\",\n \"(DavidM)-[:ACTED_IN {roles:['Brutus \\\"Brutal\\\" Howell']}]->(TheGreenMile),\\n\",\n \"(BonnieH)-[:ACTED_IN {roles:['Jan Edgecomb']}]->(TheGreenMile),\\n\",\n \"(JamesC)-[:ACTED_IN {roles:['Warden Hal Moores']}]->(TheGreenMile),\\n\",\n \"(SamR)-[:ACTED_IN {roles:['\\\"Wild Bill\\\" Wharton']}]->(TheGreenMile),\\n\",\n \"(GaryS)-[:ACTED_IN {roles:['Burt Hammersmith']}]->(TheGreenMile),\\n\",\n \"(PatriciaC)-[:ACTED_IN {roles:['Melinda Moores']}]->(TheGreenMile),\\n\",\n \"(FrankD)-[:DIRECTED]->(TheGreenMile)\\n\",\n \"\\n\",\n \"CREATE (FrostNixon:Movie {title:'Frost/Nixon', released:2008, tagline:'400 million people were waiting for the truth.'})\\n\",\n \"CREATE (FrankL:Person {name:'Frank Langella', born:1938})\\n\",\n \"CREATE (MichaelS:Person {name:'Michael Sheen', born:1969})\\n\",\n \"CREATE (OliverP:Person {name:'Oliver Platt', born:1960})\\n\",\n \"CREATE\\n\",\n \"(FrankL)-[:ACTED_IN {roles:['Richard Nixon']}]->(FrostNixon),\\n\",\n \"(MichaelS)-[:ACTED_IN {roles:['David Frost']}]->(FrostNixon),\\n\",\n \"(KevinB)-[:ACTED_IN {roles:['Jack Brennan']}]->(FrostNixon),\\n\",\n \"(OliverP)-[:ACTED_IN {roles:['Bob Zelnick']}]->(FrostNixon),\\n\",\n \"(SamR)-[:ACTED_IN {roles:['James Reston, Jr.']}]->(FrostNixon),\\n\",\n \"(RonH)-[:DIRECTED]->(FrostNixon)\\n\",\n \"\\n\",\n \"CREATE (Hoffa:Movie {title:'Hoffa', released:1992, tagline:\\\"He didn't want law. He wanted justice.\\\"})\\n\",\n \"CREATE (DannyD:Person {name:'Danny DeVito', born:1944})\\n\",\n \"CREATE (JohnR:Person {name:'John C. Reilly', born:1965})\\n\",\n \"CREATE\\n\",\n \"(JackN)-[:ACTED_IN {roles:['Hoffa']}]->(Hoffa),\\n\",\n \"(DannyD)-[:ACTED_IN {roles:['Robert \\\"Bobby\\\" Ciaro']}]->(Hoffa),\\n\",\n \"(JTW)-[:ACTED_IN {roles:['Frank Fitzsimmons']}]->(Hoffa),\\n\",\n \"(JohnR)-[:ACTED_IN {roles:['Peter \\\"Pete\\\" Connelly']}]->(Hoffa),\\n\",\n \"(DannyD)-[:DIRECTED]->(Hoffa)\\n\",\n \"\\n\",\n \"CREATE (Apollo13:Movie {title:'Apollo 13', released:1995, tagline:'Houston, we have a problem.'})\\n\",\n \"CREATE (EdH:Person {name:'Ed Harris', born:1950})\\n\",\n \"CREATE (BillPax:Person {name:'Bill Paxton', born:1955})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Jim Lovell']}]->(Apollo13),\\n\",\n \"(KevinB)-[:ACTED_IN {roles:['Jack Swigert']}]->(Apollo13),\\n\",\n \"(EdH)-[:ACTED_IN {roles:['Gene Kranz']}]->(Apollo13),\\n\",\n \"(BillPax)-[:ACTED_IN {roles:['Fred Haise']}]->(Apollo13),\\n\",\n \"(GaryS)-[:ACTED_IN {roles:['Ken Mattingly']}]->(Apollo13),\\n\",\n \"(RonH)-[:DIRECTED]->(Apollo13)\\n\",\n \"\\n\",\n \"CREATE (Twister:Movie {title:'Twister', released:1996, tagline:\\\"Don't Breathe. Don't Look Back.\\\"})\\n\",\n \"CREATE (PhilipH:Person {name:'Philip Seymour Hoffman', born:1967})\\n\",\n \"CREATE (JanB:Person {name:'Jan de Bont', born:1943})\\n\",\n \"CREATE\\n\",\n \"(BillPax)-[:ACTED_IN {roles:['Bill Harding']}]->(Twister),\\n\",\n \"(HelenH)-[:ACTED_IN {roles:['Dr. Jo Harding']}]->(Twister),\\n\",\n \"(ZachG)-[:ACTED_IN {roles:['Eddie']}]->(Twister),\\n\",\n \"(PhilipH)-[:ACTED_IN {roles:['Dustin \\\"Dusty\\\" Davis']}]->(Twister),\\n\",\n \"(JanB)-[:DIRECTED]->(Twister)\\n\",\n \"\\n\",\n \"CREATE (CastAway:Movie {title:'Cast Away', released:2000, tagline:'At the edge of the world, his journey begins.'})\\n\",\n \"CREATE (RobertZ:Person {name:'Robert Zemeckis', born:1951})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Chuck Noland']}]->(CastAway),\\n\",\n \"(HelenH)-[:ACTED_IN {roles:['Kelly Frears']}]->(CastAway),\\n\",\n \"(RobertZ)-[:DIRECTED]->(CastAway)\\n\",\n \"\\n\",\n \"CREATE (OneFlewOvertheCuckoosNest:Movie {title:\\\"One Flew Over the Cuckoo's Nest\\\", released:1975, tagline:\\\"If he's crazy, what does that make you?\\\"})\\n\",\n \"CREATE (MilosF:Person {name:'Milos Forman', born:1932})\\n\",\n \"CREATE\\n\",\n \"(JackN)-[:ACTED_IN {roles:['Randle McMurphy']}]->(OneFlewOvertheCuckoosNest),\\n\",\n \"(DannyD)-[:ACTED_IN {roles:['Martini']}]->(OneFlewOvertheCuckoosNest),\\n\",\n \"(MilosF)-[:DIRECTED]->(OneFlewOvertheCuckoosNest)\\n\",\n \"\\n\",\n \"CREATE (SomethingsGottaGive:Movie {title:\\\"Something's Gotta Give\\\", released:2003})\\n\",\n \"CREATE (DianeK:Person {name:'Diane Keaton', born:1946})\\n\",\n \"CREATE (NancyM:Person {name:'Nancy Meyers', born:1949})\\n\",\n \"CREATE\\n\",\n \"(JackN)-[:ACTED_IN {roles:['Harry Sanborn']}]->(SomethingsGottaGive),\\n\",\n \"(DianeK)-[:ACTED_IN {roles:['Erica Barry']}]->(SomethingsGottaGive),\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Julian Mercer']}]->(SomethingsGottaGive),\\n\",\n \"(NancyM)-[:DIRECTED]->(SomethingsGottaGive),\\n\",\n \"(NancyM)-[:PRODUCED]->(SomethingsGottaGive),\\n\",\n \"(NancyM)-[:WROTE]->(SomethingsGottaGive)\\n\",\n \"\\n\",\n \"CREATE (BicentennialMan:Movie {title:'Bicentennial Man', released:1999, tagline:\\\"One robot's 200 year journey to become an ordinary man.\\\"})\\n\",\n \"CREATE (ChrisC:Person {name:'Chris Columbus', born:1958})\\n\",\n \"CREATE\\n\",\n \"(Robin)-[:ACTED_IN {roles:['Andrew Marin']}]->(BicentennialMan),\\n\",\n \"(OliverP)-[:ACTED_IN {roles:['Rupert Burns']}]->(BicentennialMan),\\n\",\n \"(ChrisC)-[:DIRECTED]->(BicentennialMan)\\n\",\n \"\\n\",\n \"CREATE (CharlieWilsonsWar:Movie {title:\\\"Charlie Wilson's War\\\", released:2007, tagline:\\\"A stiff drink. A little mascara. A lot of nerve. Who said they couldn't bring down the Soviet empire.\\\"})\\n\",\n \"CREATE (JuliaR:Person {name:'Julia Roberts', born:1967})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Rep. Charlie Wilson']}]->(CharlieWilsonsWar),\\n\",\n \"(JuliaR)-[:ACTED_IN {roles:['Joanne Herring']}]->(CharlieWilsonsWar),\\n\",\n \"(PhilipH)-[:ACTED_IN {roles:['Gust Avrakotos']}]->(CharlieWilsonsWar),\\n\",\n \"(MikeN)-[:DIRECTED]->(CharlieWilsonsWar)\\n\",\n \"\\n\",\n \"CREATE (ThePolarExpress:Movie {title:'The Polar Express', released:2004, tagline:'This Holiday Season... Believe'})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Hero Boy', 'Father', 'Conductor', 'Hobo', 'Scrooge', 'Santa Claus']}]->(ThePolarExpress),\\n\",\n \"(RobertZ)-[:DIRECTED]->(ThePolarExpress)\\n\",\n \"\\n\",\n \"CREATE (ALeagueofTheirOwn:Movie {title:'A League of Their Own', released:1992, tagline:'Once in a lifetime you get a chance to do something different.'})\\n\",\n \"CREATE (Madonna:Person {name:'Madonna', born:1954})\\n\",\n \"CREATE (GeenaD:Person {name:'Geena Davis', born:1956})\\n\",\n \"CREATE (LoriP:Person {name:'Lori Petty', born:1963})\\n\",\n \"CREATE (PennyM:Person {name:'Penny Marshall', born:1943})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Jimmy Dugan']}]->(ALeagueofTheirOwn),\\n\",\n \"(GeenaD)-[:ACTED_IN {roles:['Dottie Hinson']}]->(ALeagueofTheirOwn),\\n\",\n \"(LoriP)-[:ACTED_IN {roles:['Kit Keller']}]->(ALeagueofTheirOwn),\\n\",\n \"(RosieO)-[:ACTED_IN {roles:['Doris Murphy']}]->(ALeagueofTheirOwn),\\n\",\n \"(Madonna)-[:ACTED_IN {roles:['\\\"All the Way\\\" Mae Mordabito']}]->(ALeagueofTheirOwn),\\n\",\n \"(BillPax)-[:ACTED_IN {roles:['Bob Hinson']}]->(ALeagueofTheirOwn),\\n\",\n \"(PennyM)-[:DIRECTED]->(ALeagueofTheirOwn)\\n\",\n \"\\n\",\n \"CREATE (PaulBlythe:Person {name:'Paul Blythe'})\\n\",\n \"CREATE (AngelaScope:Person {name:'Angela Scope'})\\n\",\n \"CREATE (JessicaThompson:Person {name:'Jessica Thompson'})\\n\",\n \"CREATE (JamesThompson:Person {name:'James Thompson'})\\n\",\n \"\\n\",\n \"CREATE\\n\",\n \"(JamesThompson)-[:FOLLOWS]->(JessicaThompson),\\n\",\n \"(AngelaScope)-[:FOLLOWS]->(JessicaThompson),\\n\",\n \"(PaulBlythe)-[:FOLLOWS]->(AngelaScope)\\n\",\n \"\\n\",\n \"CREATE\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:'An amazing journey', rating:95}]->(CloudAtlas),\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:'Silly, but fun', rating:65}]->(TheReplacements),\\n\",\n \"(JamesThompson)-[:REVIEWED {summary:'The coolest football movie ever', rating:100}]->(TheReplacements),\\n\",\n \"(AngelaScope)-[:REVIEWED {summary:'Pretty funny at times', rating:62}]->(TheReplacements),\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:'Dark, but compelling', rating:85}]->(Unforgiven),\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:\\\"Slapstick redeemed only by the Robin Williams and Gene Hackman's stellar performances\\\", rating:45}]->(TheBirdcage),\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:'A solid romp', rating:68}]->(TheDaVinciCode),\\n\",\n \"(JamesThompson)-[:REVIEWED {summary:'Fun, but a little far fetched', rating:65}]->(TheDaVinciCode),\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:'You had me at Jerry', rating:92}]->(JerryMaguire)\\n\",\n \"\\n\",\n \"WITH TomH as a\\n\",\n \"MATCH (a)-[:ACTED_IN]->(m)<-[:DIRECTED]-(d) RETURN a,m,d LIMIT 10;\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\\n\",\n \"\\n\",\n \"This will create the following graph model\\n\",\n \"\\n\",\n \"\"\n ],\n \"metadata\": {\n \"id\": \"AQhqv93Mj0Ss\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher -u $neo4j_url -us $neo4j_user -pw $neo4j_password\\n\",\n \"// the following Cypher query is the same as above\\n\",\n \"// and is required for running the notebook\\n\",\n \"CREATE (TheMatrix:Movie {title:'The Matrix', released:1999, tagline:'Welcome to the Real World'})\\n\",\n \"CREATE (Keanu:Person {name:'Keanu Reeves', born:1964})\\n\",\n \"CREATE (Carrie:Person {name:'Carrie-Anne Moss', born:1967})\\n\",\n \"CREATE (Laurence:Person {name:'Laurence Fishburne', born:1961})\\n\",\n \"CREATE (Hugo:Person {name:'Hugo Weaving', born:1960})\\n\",\n \"CREATE (LillyW:Person {name:'Lilly Wachowski', born:1967})\\n\",\n \"CREATE (LanaW:Person {name:'Lana Wachowski', born:1965})\\n\",\n \"CREATE (JoelS:Person {name:'Joel Silver', born:1952})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Neo']}]->(TheMatrix),\\n\",\n \"(Carrie)-[:ACTED_IN {roles:['Trinity']}]->(TheMatrix),\\n\",\n \"(Laurence)-[:ACTED_IN {roles:['Morpheus']}]->(TheMatrix),\\n\",\n \"(Hugo)-[:ACTED_IN {roles:['Agent Smith']}]->(TheMatrix),\\n\",\n \"(LillyW)-[:DIRECTED]->(TheMatrix),\\n\",\n \"(LanaW)-[:DIRECTED]->(TheMatrix),\\n\",\n \"(JoelS)-[:PRODUCED]->(TheMatrix)\\n\",\n \"\\n\",\n \"CREATE (Emil:Person {name:\\\"Emil Eifrem\\\", born:1978})\\n\",\n \"CREATE (Emil)-[:ACTED_IN {roles:[\\\"Emil\\\"]}]->(TheMatrix)\\n\",\n \"\\n\",\n \"CREATE (TheMatrixReloaded:Movie {title:'The Matrix Reloaded', released:2003, tagline:'Free your mind'})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Neo']}]->(TheMatrixReloaded),\\n\",\n \"(Carrie)-[:ACTED_IN {roles:['Trinity']}]->(TheMatrixReloaded),\\n\",\n \"(Laurence)-[:ACTED_IN {roles:['Morpheus']}]->(TheMatrixReloaded),\\n\",\n \"(Hugo)-[:ACTED_IN {roles:['Agent Smith']}]->(TheMatrixReloaded),\\n\",\n \"(LillyW)-[:DIRECTED]->(TheMatrixReloaded),\\n\",\n \"(LanaW)-[:DIRECTED]->(TheMatrixReloaded),\\n\",\n \"(JoelS)-[:PRODUCED]->(TheMatrixReloaded)\\n\",\n \"\\n\",\n \"CREATE (TheMatrixRevolutions:Movie {title:'The Matrix Revolutions', released:2003, tagline:'Everything that has a beginning has an end'})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Neo']}]->(TheMatrixRevolutions),\\n\",\n \"(Carrie)-[:ACTED_IN {roles:['Trinity']}]->(TheMatrixRevolutions),\\n\",\n \"(Laurence)-[:ACTED_IN {roles:['Morpheus']}]->(TheMatrixRevolutions),\\n\",\n \"(Hugo)-[:ACTED_IN {roles:['Agent Smith']}]->(TheMatrixRevolutions),\\n\",\n \"(LillyW)-[:DIRECTED]->(TheMatrixRevolutions),\\n\",\n \"(LanaW)-[:DIRECTED]->(TheMatrixRevolutions),\\n\",\n \"(JoelS)-[:PRODUCED]->(TheMatrixRevolutions)\\n\",\n \"\\n\",\n \"CREATE (TheDevilsAdvocate:Movie {title:\\\"The Devil's Advocate\\\", released:1997, tagline:'Evil has its winning ways'})\\n\",\n \"CREATE (Charlize:Person {name:'Charlize Theron', born:1975})\\n\",\n \"CREATE (Al:Person {name:'Al Pacino', born:1940})\\n\",\n \"CREATE (Taylor:Person {name:'Taylor Hackford', born:1944})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Kevin Lomax']}]->(TheDevilsAdvocate),\\n\",\n \"(Charlize)-[:ACTED_IN {roles:['Mary Ann Lomax']}]->(TheDevilsAdvocate),\\n\",\n \"(Al)-[:ACTED_IN {roles:['John Milton']}]->(TheDevilsAdvocate),\\n\",\n \"(Taylor)-[:DIRECTED]->(TheDevilsAdvocate)\\n\",\n \"\\n\",\n \"CREATE (AFewGoodMen:Movie {title:\\\"A Few Good Men\\\", released:1992, tagline:\\\"In the heart of the nation's capital, in a courthouse of the U.S. government, one man will stop at nothing to keep his honor, and one will stop at nothing to find the truth.\\\"})\\n\",\n \"CREATE (TomC:Person {name:'Tom Cruise', born:1962})\\n\",\n \"CREATE (JackN:Person {name:'Jack Nicholson', born:1937})\\n\",\n \"CREATE (DemiM:Person {name:'Demi Moore', born:1962})\\n\",\n \"CREATE (KevinB:Person {name:'Kevin Bacon', born:1958})\\n\",\n \"CREATE (KieferS:Person {name:'Kiefer Sutherland', born:1966})\\n\",\n \"CREATE (NoahW:Person {name:'Noah Wyle', born:1971})\\n\",\n \"CREATE (CubaG:Person {name:'Cuba Gooding Jr.', born:1968})\\n\",\n \"CREATE (KevinP:Person {name:'Kevin Pollak', born:1957})\\n\",\n \"CREATE (JTW:Person {name:'J.T. Walsh', born:1943})\\n\",\n \"CREATE (JamesM:Person {name:'James Marshall', born:1967})\\n\",\n \"CREATE (ChristopherG:Person {name:'Christopher Guest', born:1948})\\n\",\n \"CREATE (RobR:Person {name:'Rob Reiner', born:1947})\\n\",\n \"CREATE (AaronS:Person {name:'Aaron Sorkin', born:1961})\\n\",\n \"CREATE\\n\",\n \"(TomC)-[:ACTED_IN {roles:['Lt. Daniel Kaffee']}]->(AFewGoodMen),\\n\",\n \"(JackN)-[:ACTED_IN {roles:['Col. Nathan R. Jessup']}]->(AFewGoodMen),\\n\",\n \"(DemiM)-[:ACTED_IN {roles:['Lt. Cdr. JoAnne Galloway']}]->(AFewGoodMen),\\n\",\n \"(KevinB)-[:ACTED_IN {roles:['Capt. Jack Ross']}]->(AFewGoodMen),\\n\",\n \"(KieferS)-[:ACTED_IN {roles:['Lt. Jonathan Kendrick']}]->(AFewGoodMen),\\n\",\n \"(NoahW)-[:ACTED_IN {roles:['Cpl. Jeffrey Barnes']}]->(AFewGoodMen),\\n\",\n \"(CubaG)-[:ACTED_IN {roles:['Cpl. Carl Hammaker']}]->(AFewGoodMen),\\n\",\n \"(KevinP)-[:ACTED_IN {roles:['Lt. Sam Weinberg']}]->(AFewGoodMen),\\n\",\n \"(JTW)-[:ACTED_IN {roles:['Lt. Col. Matthew Andrew Markinson']}]->(AFewGoodMen),\\n\",\n \"(JamesM)-[:ACTED_IN {roles:['Pfc. Louden Downey']}]->(AFewGoodMen),\\n\",\n \"(ChristopherG)-[:ACTED_IN {roles:['Dr. Stone']}]->(AFewGoodMen),\\n\",\n \"(AaronS)-[:ACTED_IN {roles:['Man in Bar']}]->(AFewGoodMen),\\n\",\n \"(RobR)-[:DIRECTED]->(AFewGoodMen),\\n\",\n \"(AaronS)-[:WROTE]->(AFewGoodMen)\\n\",\n \"\\n\",\n \"CREATE (TopGun:Movie {title:\\\"Top Gun\\\", released:1986, tagline:'I feel the need, the need for speed.'})\\n\",\n \"CREATE (KellyM:Person {name:'Kelly McGillis', born:1957})\\n\",\n \"CREATE (ValK:Person {name:'Val Kilmer', born:1959})\\n\",\n \"CREATE (AnthonyE:Person {name:'Anthony Edwards', born:1962})\\n\",\n \"CREATE (TomS:Person {name:'Tom Skerritt', born:1933})\\n\",\n \"CREATE (MegR:Person {name:'Meg Ryan', born:1961})\\n\",\n \"CREATE (TonyS:Person {name:'Tony Scott', born:1944})\\n\",\n \"CREATE (JimC:Person {name:'Jim Cash', born:1941})\\n\",\n \"CREATE\\n\",\n \"(TomC)-[:ACTED_IN {roles:['Maverick']}]->(TopGun),\\n\",\n \"(KellyM)-[:ACTED_IN {roles:['Charlie']}]->(TopGun),\\n\",\n \"(ValK)-[:ACTED_IN {roles:['Iceman']}]->(TopGun),\\n\",\n \"(AnthonyE)-[:ACTED_IN {roles:['Goose']}]->(TopGun),\\n\",\n \"(TomS)-[:ACTED_IN {roles:['Viper']}]->(TopGun),\\n\",\n \"(MegR)-[:ACTED_IN {roles:['Carole']}]->(TopGun),\\n\",\n \"(TonyS)-[:DIRECTED]->(TopGun),\\n\",\n \"(JimC)-[:WROTE]->(TopGun)\\n\",\n \"\\n\",\n \"CREATE (JerryMaguire:Movie {title:'Jerry Maguire', released:2000, tagline:'The rest of his life begins now.'})\\n\",\n \"CREATE (ReneeZ:Person {name:'Renee Zellweger', born:1969})\\n\",\n \"CREATE (KellyP:Person {name:'Kelly Preston', born:1962})\\n\",\n \"CREATE (JerryO:Person {name:\\\"Jerry O'Connell\\\", born:1974})\\n\",\n \"CREATE (JayM:Person {name:'Jay Mohr', born:1970})\\n\",\n \"CREATE (BonnieH:Person {name:'Bonnie Hunt', born:1961})\\n\",\n \"CREATE (ReginaK:Person {name:'Regina King', born:1971})\\n\",\n \"CREATE (JonathanL:Person {name:'Jonathan Lipnicki', born:1996})\\n\",\n \"CREATE (CameronC:Person {name:'Cameron Crowe', born:1957})\\n\",\n \"CREATE\\n\",\n \"(TomC)-[:ACTED_IN {roles:['Jerry Maguire']}]->(JerryMaguire),\\n\",\n \"(CubaG)-[:ACTED_IN {roles:['Rod Tidwell']}]->(JerryMaguire),\\n\",\n \"(ReneeZ)-[:ACTED_IN {roles:['Dorothy Boyd']}]->(JerryMaguire),\\n\",\n \"(KellyP)-[:ACTED_IN {roles:['Avery Bishop']}]->(JerryMaguire),\\n\",\n \"(JerryO)-[:ACTED_IN {roles:['Frank Cushman']}]->(JerryMaguire),\\n\",\n \"(JayM)-[:ACTED_IN {roles:['Bob Sugar']}]->(JerryMaguire),\\n\",\n \"(BonnieH)-[:ACTED_IN {roles:['Laurel Boyd']}]->(JerryMaguire),\\n\",\n \"(ReginaK)-[:ACTED_IN {roles:['Marcee Tidwell']}]->(JerryMaguire),\\n\",\n \"(JonathanL)-[:ACTED_IN {roles:['Ray Boyd']}]->(JerryMaguire),\\n\",\n \"(CameronC)-[:DIRECTED]->(JerryMaguire),\\n\",\n \"(CameronC)-[:PRODUCED]->(JerryMaguire),\\n\",\n \"(CameronC)-[:WROTE]->(JerryMaguire)\\n\",\n \"\\n\",\n \"CREATE (StandByMe:Movie {title:\\\"Stand By Me\\\", released:1986, tagline:\\\"For some, it's the last real taste of innocence, and the first real taste of life. But for everyone, it's the time that memories are made of.\\\"})\\n\",\n \"CREATE (RiverP:Person {name:'River Phoenix', born:1970})\\n\",\n \"CREATE (CoreyF:Person {name:'Corey Feldman', born:1971})\\n\",\n \"CREATE (WilW:Person {name:'Wil Wheaton', born:1972})\\n\",\n \"CREATE (JohnC:Person {name:'John Cusack', born:1966})\\n\",\n \"CREATE (MarshallB:Person {name:'Marshall Bell', born:1942})\\n\",\n \"CREATE\\n\",\n \"(WilW)-[:ACTED_IN {roles:['Gordie Lachance']}]->(StandByMe),\\n\",\n \"(RiverP)-[:ACTED_IN {roles:['Chris Chambers']}]->(StandByMe),\\n\",\n \"(JerryO)-[:ACTED_IN {roles:['Vern Tessio']}]->(StandByMe),\\n\",\n \"(CoreyF)-[:ACTED_IN {roles:['Teddy Duchamp']}]->(StandByMe),\\n\",\n \"(JohnC)-[:ACTED_IN {roles:['Denny Lachance']}]->(StandByMe),\\n\",\n \"(KieferS)-[:ACTED_IN {roles:['Ace Merrill']}]->(StandByMe),\\n\",\n \"(MarshallB)-[:ACTED_IN {roles:['Mr. Lachance']}]->(StandByMe),\\n\",\n \"(RobR)-[:DIRECTED]->(StandByMe)\\n\",\n \"\\n\",\n \"CREATE (AsGoodAsItGets:Movie {title:'As Good as It Gets', released:1997, tagline:'A comedy from the heart that goes for the throat.'})\\n\",\n \"CREATE (HelenH:Person {name:'Helen Hunt', born:1963})\\n\",\n \"CREATE (GregK:Person {name:'Greg Kinnear', born:1963})\\n\",\n \"CREATE (JamesB:Person {name:'James L. Brooks', born:1940})\\n\",\n \"CREATE\\n\",\n \"(JackN)-[:ACTED_IN {roles:['Melvin Udall']}]->(AsGoodAsItGets),\\n\",\n \"(HelenH)-[:ACTED_IN {roles:['Carol Connelly']}]->(AsGoodAsItGets),\\n\",\n \"(GregK)-[:ACTED_IN {roles:['Simon Bishop']}]->(AsGoodAsItGets),\\n\",\n \"(CubaG)-[:ACTED_IN {roles:['Frank Sachs']}]->(AsGoodAsItGets),\\n\",\n \"(JamesB)-[:DIRECTED]->(AsGoodAsItGets)\\n\",\n \"\\n\",\n \"CREATE (WhatDreamsMayCome:Movie {title:'What Dreams May Come', released:1998, tagline:'After life there is more. The end is just the beginning.'})\\n\",\n \"CREATE (AnnabellaS:Person {name:'Annabella Sciorra', born:1960})\\n\",\n \"CREATE (MaxS:Person {name:'Max von Sydow', born:1929})\\n\",\n \"CREATE (WernerH:Person {name:'Werner Herzog', born:1942})\\n\",\n \"CREATE (Robin:Person {name:'Robin Williams', born:1951})\\n\",\n \"CREATE (VincentW:Person {name:'Vincent Ward', born:1956})\\n\",\n \"CREATE\\n\",\n \"(Robin)-[:ACTED_IN {roles:['Chris Nielsen']}]->(WhatDreamsMayCome),\\n\",\n \"(CubaG)-[:ACTED_IN {roles:['Albert Lewis']}]->(WhatDreamsMayCome),\\n\",\n \"(AnnabellaS)-[:ACTED_IN {roles:['Annie Collins-Nielsen']}]->(WhatDreamsMayCome),\\n\",\n \"(MaxS)-[:ACTED_IN {roles:['The Tracker']}]->(WhatDreamsMayCome),\\n\",\n \"(WernerH)-[:ACTED_IN {roles:['The Face']}]->(WhatDreamsMayCome),\\n\",\n \"(VincentW)-[:DIRECTED]->(WhatDreamsMayCome)\\n\",\n \"\\n\",\n \"CREATE (SnowFallingonCedars:Movie {title:'Snow Falling on Cedars', released:1999, tagline:'First loves last. Forever.'})\\n\",\n \"CREATE (EthanH:Person {name:'Ethan Hawke', born:1970})\\n\",\n \"CREATE (RickY:Person {name:'Rick Yune', born:1971})\\n\",\n \"CREATE (JamesC:Person {name:'James Cromwell', born:1940})\\n\",\n \"CREATE (ScottH:Person {name:'Scott Hicks', born:1953})\\n\",\n \"CREATE\\n\",\n \"(EthanH)-[:ACTED_IN {roles:['Ishmael Chambers']}]->(SnowFallingonCedars),\\n\",\n \"(RickY)-[:ACTED_IN {roles:['Kazuo Miyamoto']}]->(SnowFallingonCedars),\\n\",\n \"(MaxS)-[:ACTED_IN {roles:['Nels Gudmundsson']}]->(SnowFallingonCedars),\\n\",\n \"(JamesC)-[:ACTED_IN {roles:['Judge Fielding']}]->(SnowFallingonCedars),\\n\",\n \"(ScottH)-[:DIRECTED]->(SnowFallingonCedars)\\n\",\n \"\\n\",\n \"CREATE (YouveGotMail:Movie {title:\\\"You've Got Mail\\\", released:1998, tagline:'At odds in life... in love on-line.'})\\n\",\n \"CREATE (ParkerP:Person {name:'Parker Posey', born:1968})\\n\",\n \"CREATE (DaveC:Person {name:'Dave Chappelle', born:1973})\\n\",\n \"CREATE (SteveZ:Person {name:'Steve Zahn', born:1967})\\n\",\n \"CREATE (TomH:Person {name:'Tom Hanks', born:1956})\\n\",\n \"CREATE (NoraE:Person {name:'Nora Ephron', born:1941})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Joe Fox']}]->(YouveGotMail),\\n\",\n \"(MegR)-[:ACTED_IN {roles:['Kathleen Kelly']}]->(YouveGotMail),\\n\",\n \"(GregK)-[:ACTED_IN {roles:['Frank Navasky']}]->(YouveGotMail),\\n\",\n \"(ParkerP)-[:ACTED_IN {roles:['Patricia Eden']}]->(YouveGotMail),\\n\",\n \"(DaveC)-[:ACTED_IN {roles:['Kevin Jackson']}]->(YouveGotMail),\\n\",\n \"(SteveZ)-[:ACTED_IN {roles:['George Pappas']}]->(YouveGotMail),\\n\",\n \"(NoraE)-[:DIRECTED]->(YouveGotMail)\\n\",\n \"\\n\",\n \"CREATE (SleeplessInSeattle:Movie {title:'Sleepless in Seattle', released:1993, tagline:'What if someone you never met, someone you never saw, someone you never knew was the only someone for you?'})\\n\",\n \"CREATE (RitaW:Person {name:'Rita Wilson', born:1956})\\n\",\n \"CREATE (BillPull:Person {name:'Bill Pullman', born:1953})\\n\",\n \"CREATE (VictorG:Person {name:'Victor Garber', born:1949})\\n\",\n \"CREATE (RosieO:Person {name:\\\"Rosie O'Donnell\\\", born:1962})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Sam Baldwin']}]->(SleeplessInSeattle),\\n\",\n \"(MegR)-[:ACTED_IN {roles:['Annie Reed']}]->(SleeplessInSeattle),\\n\",\n \"(RitaW)-[:ACTED_IN {roles:['Suzy']}]->(SleeplessInSeattle),\\n\",\n \"(BillPull)-[:ACTED_IN {roles:['Walter']}]->(SleeplessInSeattle),\\n\",\n \"(VictorG)-[:ACTED_IN {roles:['Greg']}]->(SleeplessInSeattle),\\n\",\n \"(RosieO)-[:ACTED_IN {roles:['Becky']}]->(SleeplessInSeattle),\\n\",\n \"(NoraE)-[:DIRECTED]->(SleeplessInSeattle)\\n\",\n \"\\n\",\n \"CREATE (JoeVersustheVolcano:Movie {title:'Joe Versus the Volcano', released:1990, tagline:'A story of love, lava and burning desire.'})\\n\",\n \"CREATE (JohnS:Person {name:'John Patrick Stanley', born:1950})\\n\",\n \"CREATE (Nathan:Person {name:'Nathan Lane', born:1956})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Joe Banks']}]->(JoeVersustheVolcano),\\n\",\n \"(MegR)-[:ACTED_IN {roles:['DeDe', 'Angelica Graynamore', 'Patricia Graynamore']}]->(JoeVersustheVolcano),\\n\",\n \"(Nathan)-[:ACTED_IN {roles:['Baw']}]->(JoeVersustheVolcano),\\n\",\n \"(JohnS)-[:DIRECTED]->(JoeVersustheVolcano)\\n\",\n \"\\n\",\n \"CREATE (WhenHarryMetSally:Movie {title:'When Harry Met Sally', released:1998, tagline:'Can two friends sleep together and still love each other in the morning?'})\\n\",\n \"CREATE (BillyC:Person {name:'Billy Crystal', born:1948})\\n\",\n \"CREATE (CarrieF:Person {name:'Carrie Fisher', born:1956})\\n\",\n \"CREATE (BrunoK:Person {name:'Bruno Kirby', born:1949})\\n\",\n \"CREATE\\n\",\n \"(BillyC)-[:ACTED_IN {roles:['Harry Burns']}]->(WhenHarryMetSally),\\n\",\n \"(MegR)-[:ACTED_IN {roles:['Sally Albright']}]->(WhenHarryMetSally),\\n\",\n \"(CarrieF)-[:ACTED_IN {roles:['Marie']}]->(WhenHarryMetSally),\\n\",\n \"(BrunoK)-[:ACTED_IN {roles:['Jess']}]->(WhenHarryMetSally),\\n\",\n \"(RobR)-[:DIRECTED]->(WhenHarryMetSally),\\n\",\n \"(RobR)-[:PRODUCED]->(WhenHarryMetSally),\\n\",\n \"(NoraE)-[:PRODUCED]->(WhenHarryMetSally),\\n\",\n \"(NoraE)-[:WROTE]->(WhenHarryMetSally)\\n\",\n \"\\n\",\n \"CREATE (ThatThingYouDo:Movie {title:'That Thing You Do', released:1996, tagline:'In every life there comes a time when that thing you dream becomes that thing you do'})\\n\",\n \"CREATE (LivT:Person {name:'Liv Tyler', born:1977})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Mr. White']}]->(ThatThingYouDo),\\n\",\n \"(LivT)-[:ACTED_IN {roles:['Faye Dolan']}]->(ThatThingYouDo),\\n\",\n \"(Charlize)-[:ACTED_IN {roles:['Tina']}]->(ThatThingYouDo),\\n\",\n \"(TomH)-[:DIRECTED]->(ThatThingYouDo)\\n\",\n \"\\n\",\n \"CREATE (TheReplacements:Movie {title:'The Replacements', released:2000, tagline:'Pain heals, Chicks dig scars... Glory lasts forever'})\\n\",\n \"CREATE (Brooke:Person {name:'Brooke Langton', born:1970})\\n\",\n \"CREATE (Gene:Person {name:'Gene Hackman', born:1930})\\n\",\n \"CREATE (Orlando:Person {name:'Orlando Jones', born:1968})\\n\",\n \"CREATE (Howard:Person {name:'Howard Deutch', born:1950})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Shane Falco']}]->(TheReplacements),\\n\",\n \"(Brooke)-[:ACTED_IN {roles:['Annabelle Farrell']}]->(TheReplacements),\\n\",\n \"(Gene)-[:ACTED_IN {roles:['Jimmy McGinty']}]->(TheReplacements),\\n\",\n \"(Orlando)-[:ACTED_IN {roles:['Clifford Franklin']}]->(TheReplacements),\\n\",\n \"(Howard)-[:DIRECTED]->(TheReplacements)\\n\",\n \"\\n\",\n \"CREATE (RescueDawn:Movie {title:'RescueDawn', released:2006, tagline:\\\"Based on the extraordinary true story of one man's fight for freedom\\\"})\\n\",\n \"CREATE (ChristianB:Person {name:'Christian Bale', born:1974})\\n\",\n \"CREATE (ZachG:Person {name:'Zach Grenier', born:1954})\\n\",\n \"CREATE\\n\",\n \"(MarshallB)-[:ACTED_IN {roles:['Admiral']}]->(RescueDawn),\\n\",\n \"(ChristianB)-[:ACTED_IN {roles:['Dieter Dengler']}]->(RescueDawn),\\n\",\n \"(ZachG)-[:ACTED_IN {roles:['Squad Leader']}]->(RescueDawn),\\n\",\n \"(SteveZ)-[:ACTED_IN {roles:['Duane']}]->(RescueDawn),\\n\",\n \"(WernerH)-[:DIRECTED]->(RescueDawn)\\n\",\n \"\\n\",\n \"CREATE (TheBirdcage:Movie {title:'The Birdcage', released:1996, tagline:'Come as you are'})\\n\",\n \"CREATE (MikeN:Person {name:'Mike Nichols', born:1931})\\n\",\n \"CREATE\\n\",\n \"(Robin)-[:ACTED_IN {roles:['Armand Goldman']}]->(TheBirdcage),\\n\",\n \"(Nathan)-[:ACTED_IN {roles:['Albert Goldman']}]->(TheBirdcage),\\n\",\n \"(Gene)-[:ACTED_IN {roles:['Sen. Kevin Keeley']}]->(TheBirdcage),\\n\",\n \"(MikeN)-[:DIRECTED]->(TheBirdcage)\\n\",\n \"\\n\",\n \"CREATE (Unforgiven:Movie {title:'Unforgiven', released:1992, tagline:\\\"It's a hell of a thing, killing a man\\\"})\\n\",\n \"CREATE (RichardH:Person {name:'Richard Harris', born:1930})\\n\",\n \"CREATE (ClintE:Person {name:'Clint Eastwood', born:1930})\\n\",\n \"CREATE\\n\",\n \"(RichardH)-[:ACTED_IN {roles:['English Bob']}]->(Unforgiven),\\n\",\n \"(ClintE)-[:ACTED_IN {roles:['Bill Munny']}]->(Unforgiven),\\n\",\n \"(Gene)-[:ACTED_IN {roles:['Little Bill Daggett']}]->(Unforgiven),\\n\",\n \"(ClintE)-[:DIRECTED]->(Unforgiven)\\n\",\n \"\\n\",\n \"CREATE (JohnnyMnemonic:Movie {title:'Johnny Mnemonic', released:1995, tagline:'The hottest data on earth. In the coolest head in town'})\\n\",\n \"CREATE (Takeshi:Person {name:'Takeshi Kitano', born:1947})\\n\",\n \"CREATE (Dina:Person {name:'Dina Meyer', born:1968})\\n\",\n \"CREATE (IceT:Person {name:'Ice-T', born:1958})\\n\",\n \"CREATE (RobertL:Person {name:'Robert Longo', born:1953})\\n\",\n \"CREATE\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Johnny Mnemonic']}]->(JohnnyMnemonic),\\n\",\n \"(Takeshi)-[:ACTED_IN {roles:['Takahashi']}]->(JohnnyMnemonic),\\n\",\n \"(Dina)-[:ACTED_IN {roles:['Jane']}]->(JohnnyMnemonic),\\n\",\n \"(IceT)-[:ACTED_IN {roles:['J-Bone']}]->(JohnnyMnemonic),\\n\",\n \"(RobertL)-[:DIRECTED]->(JohnnyMnemonic)\\n\",\n \"\\n\",\n \"CREATE (CloudAtlas:Movie {title:'Cloud Atlas', released:2012, tagline:'Everything is connected'})\\n\",\n \"CREATE (HalleB:Person {name:'Halle Berry', born:1966})\\n\",\n \"CREATE (JimB:Person {name:'Jim Broadbent', born:1949})\\n\",\n \"CREATE (TomT:Person {name:'Tom Tykwer', born:1965})\\n\",\n \"CREATE (DavidMitchell:Person {name:'David Mitchell', born:1969})\\n\",\n \"CREATE (StefanArndt:Person {name:'Stefan Arndt', born:1961})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Zachry', 'Dr. Henry Goose', 'Isaac Sachs', 'Dermot Hoggins']}]->(CloudAtlas),\\n\",\n \"(Hugo)-[:ACTED_IN {roles:['Bill Smoke', 'Haskell Moore', 'Tadeusz Kesselring', 'Nurse Noakes', 'Boardman Mephi', 'Old Georgie']}]->(CloudAtlas),\\n\",\n \"(HalleB)-[:ACTED_IN {roles:['Luisa Rey', 'Jocasta Ayrs', 'Ovid', 'Meronym']}]->(CloudAtlas),\\n\",\n \"(JimB)-[:ACTED_IN {roles:['Vyvyan Ayrs', 'Captain Molyneux', 'Timothy Cavendish']}]->(CloudAtlas),\\n\",\n \"(TomT)-[:DIRECTED]->(CloudAtlas),\\n\",\n \"(LillyW)-[:DIRECTED]->(CloudAtlas),\\n\",\n \"(LanaW)-[:DIRECTED]->(CloudAtlas),\\n\",\n \"(DavidMitchell)-[:WROTE]->(CloudAtlas),\\n\",\n \"(StefanArndt)-[:PRODUCED]->(CloudAtlas)\\n\",\n \"\\n\",\n \"CREATE (TheDaVinciCode:Movie {title:'The Da Vinci Code', released:2006, tagline:'Break The Codes'})\\n\",\n \"CREATE (IanM:Person {name:'Ian McKellen', born:1939})\\n\",\n \"CREATE (AudreyT:Person {name:'Audrey Tautou', born:1976})\\n\",\n \"CREATE (PaulB:Person {name:'Paul Bettany', born:1971})\\n\",\n \"CREATE (RonH:Person {name:'Ron Howard', born:1954})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Dr. Robert Langdon']}]->(TheDaVinciCode),\\n\",\n \"(IanM)-[:ACTED_IN {roles:['Sir Leight Teabing']}]->(TheDaVinciCode),\\n\",\n \"(AudreyT)-[:ACTED_IN {roles:['Sophie Neveu']}]->(TheDaVinciCode),\\n\",\n \"(PaulB)-[:ACTED_IN {roles:['Silas']}]->(TheDaVinciCode),\\n\",\n \"(RonH)-[:DIRECTED]->(TheDaVinciCode)\\n\",\n \"\\n\",\n \"CREATE (VforVendetta:Movie {title:'V for Vendetta', released:2006, tagline:'Freedom! Forever!'})\\n\",\n \"CREATE (NatalieP:Person {name:'Natalie Portman', born:1981})\\n\",\n \"CREATE (StephenR:Person {name:'Stephen Rea', born:1946})\\n\",\n \"CREATE (JohnH:Person {name:'John Hurt', born:1940})\\n\",\n \"CREATE (BenM:Person {name: 'Ben Miles', born:1967})\\n\",\n \"CREATE\\n\",\n \"(Hugo)-[:ACTED_IN {roles:['V']}]->(VforVendetta),\\n\",\n \"(NatalieP)-[:ACTED_IN {roles:['Evey Hammond']}]->(VforVendetta),\\n\",\n \"(StephenR)-[:ACTED_IN {roles:['Eric Finch']}]->(VforVendetta),\\n\",\n \"(JohnH)-[:ACTED_IN {roles:['High Chancellor Adam Sutler']}]->(VforVendetta),\\n\",\n \"(BenM)-[:ACTED_IN {roles:['Dascomb']}]->(VforVendetta),\\n\",\n \"(JamesM)-[:DIRECTED]->(VforVendetta),\\n\",\n \"(LillyW)-[:PRODUCED]->(VforVendetta),\\n\",\n \"(LanaW)-[:PRODUCED]->(VforVendetta),\\n\",\n \"(JoelS)-[:PRODUCED]->(VforVendetta),\\n\",\n \"(LillyW)-[:WROTE]->(VforVendetta),\\n\",\n \"(LanaW)-[:WROTE]->(VforVendetta)\\n\",\n \"\\n\",\n \"CREATE (SpeedRacer:Movie {title:'Speed Racer', released:2008, tagline:'Speed has no limits'})\\n\",\n \"CREATE (EmileH:Person {name:'Emile Hirsch', born:1985})\\n\",\n \"CREATE (JohnG:Person {name:'John Goodman', born:1960})\\n\",\n \"CREATE (SusanS:Person {name:'Susan Sarandon', born:1946})\\n\",\n \"CREATE (MatthewF:Person {name:'Matthew Fox', born:1966})\\n\",\n \"CREATE (ChristinaR:Person {name:'Christina Ricci', born:1980})\\n\",\n \"CREATE (Rain:Person {name:'Rain', born:1982})\\n\",\n \"CREATE\\n\",\n \"(EmileH)-[:ACTED_IN {roles:['Speed Racer']}]->(SpeedRacer),\\n\",\n \"(JohnG)-[:ACTED_IN {roles:['Pops']}]->(SpeedRacer),\\n\",\n \"(SusanS)-[:ACTED_IN {roles:['Mom']}]->(SpeedRacer),\\n\",\n \"(MatthewF)-[:ACTED_IN {roles:['Racer X']}]->(SpeedRacer),\\n\",\n \"(ChristinaR)-[:ACTED_IN {roles:['Trixie']}]->(SpeedRacer),\\n\",\n \"(Rain)-[:ACTED_IN {roles:['Taejo Togokahn']}]->(SpeedRacer),\\n\",\n \"(BenM)-[:ACTED_IN {roles:['Cass Jones']}]->(SpeedRacer),\\n\",\n \"(LillyW)-[:DIRECTED]->(SpeedRacer),\\n\",\n \"(LanaW)-[:DIRECTED]->(SpeedRacer),\\n\",\n \"(LillyW)-[:WROTE]->(SpeedRacer),\\n\",\n \"(LanaW)-[:WROTE]->(SpeedRacer),\\n\",\n \"(JoelS)-[:PRODUCED]->(SpeedRacer)\\n\",\n \"\\n\",\n \"CREATE (NinjaAssassin:Movie {title:'Ninja Assassin', released:2009, tagline:'Prepare to enter a secret world of assassins'})\\n\",\n \"CREATE (NaomieH:Person {name:'Naomie Harris'})\\n\",\n \"CREATE\\n\",\n \"(Rain)-[:ACTED_IN {roles:['Raizo']}]->(NinjaAssassin),\\n\",\n \"(NaomieH)-[:ACTED_IN {roles:['Mika Coretti']}]->(NinjaAssassin),\\n\",\n \"(RickY)-[:ACTED_IN {roles:['Takeshi']}]->(NinjaAssassin),\\n\",\n \"(BenM)-[:ACTED_IN {roles:['Ryan Maslow']}]->(NinjaAssassin),\\n\",\n \"(JamesM)-[:DIRECTED]->(NinjaAssassin),\\n\",\n \"(LillyW)-[:PRODUCED]->(NinjaAssassin),\\n\",\n \"(LanaW)-[:PRODUCED]->(NinjaAssassin),\\n\",\n \"(JoelS)-[:PRODUCED]->(NinjaAssassin)\\n\",\n \"\\n\",\n \"CREATE (TheGreenMile:Movie {title:'The Green Mile', released:1999, tagline:\\\"Walk a mile you'll never forget.\\\"})\\n\",\n \"CREATE (MichaelD:Person {name:'Michael Clarke Duncan', born:1957})\\n\",\n \"CREATE (DavidM:Person {name:'David Morse', born:1953})\\n\",\n \"CREATE (SamR:Person {name:'Sam Rockwell', born:1968})\\n\",\n \"CREATE (GaryS:Person {name:'Gary Sinise', born:1955})\\n\",\n \"CREATE (PatriciaC:Person {name:'Patricia Clarkson', born:1959})\\n\",\n \"CREATE (FrankD:Person {name:'Frank Darabont', born:1959})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Paul Edgecomb']}]->(TheGreenMile),\\n\",\n \"(MichaelD)-[:ACTED_IN {roles:['John Coffey']}]->(TheGreenMile),\\n\",\n \"(DavidM)-[:ACTED_IN {roles:['Brutus \\\"Brutal\\\" Howell']}]->(TheGreenMile),\\n\",\n \"(BonnieH)-[:ACTED_IN {roles:['Jan Edgecomb']}]->(TheGreenMile),\\n\",\n \"(JamesC)-[:ACTED_IN {roles:['Warden Hal Moores']}]->(TheGreenMile),\\n\",\n \"(SamR)-[:ACTED_IN {roles:['\\\"Wild Bill\\\" Wharton']}]->(TheGreenMile),\\n\",\n \"(GaryS)-[:ACTED_IN {roles:['Burt Hammersmith']}]->(TheGreenMile),\\n\",\n \"(PatriciaC)-[:ACTED_IN {roles:['Melinda Moores']}]->(TheGreenMile),\\n\",\n \"(FrankD)-[:DIRECTED]->(TheGreenMile)\\n\",\n \"\\n\",\n \"CREATE (FrostNixon:Movie {title:'Frost/Nixon', released:2008, tagline:'400 million people were waiting for the truth.'})\\n\",\n \"CREATE (FrankL:Person {name:'Frank Langella', born:1938})\\n\",\n \"CREATE (MichaelS:Person {name:'Michael Sheen', born:1969})\\n\",\n \"CREATE (OliverP:Person {name:'Oliver Platt', born:1960})\\n\",\n \"CREATE\\n\",\n \"(FrankL)-[:ACTED_IN {roles:['Richard Nixon']}]->(FrostNixon),\\n\",\n \"(MichaelS)-[:ACTED_IN {roles:['David Frost']}]->(FrostNixon),\\n\",\n \"(KevinB)-[:ACTED_IN {roles:['Jack Brennan']}]->(FrostNixon),\\n\",\n \"(OliverP)-[:ACTED_IN {roles:['Bob Zelnick']}]->(FrostNixon),\\n\",\n \"(SamR)-[:ACTED_IN {roles:['James Reston, Jr.']}]->(FrostNixon),\\n\",\n \"(RonH)-[:DIRECTED]->(FrostNixon)\\n\",\n \"\\n\",\n \"CREATE (Hoffa:Movie {title:'Hoffa', released:1992, tagline:\\\"He didn't want law. He wanted justice.\\\"})\\n\",\n \"CREATE (DannyD:Person {name:'Danny DeVito', born:1944})\\n\",\n \"CREATE (JohnR:Person {name:'John C. Reilly', born:1965})\\n\",\n \"CREATE\\n\",\n \"(JackN)-[:ACTED_IN {roles:['Hoffa']}]->(Hoffa),\\n\",\n \"(DannyD)-[:ACTED_IN {roles:['Robert \\\"Bobby\\\" Ciaro']}]->(Hoffa),\\n\",\n \"(JTW)-[:ACTED_IN {roles:['Frank Fitzsimmons']}]->(Hoffa),\\n\",\n \"(JohnR)-[:ACTED_IN {roles:['Peter \\\"Pete\\\" Connelly']}]->(Hoffa),\\n\",\n \"(DannyD)-[:DIRECTED]->(Hoffa)\\n\",\n \"\\n\",\n \"CREATE (Apollo13:Movie {title:'Apollo 13', released:1995, tagline:'Houston, we have a problem.'})\\n\",\n \"CREATE (EdH:Person {name:'Ed Harris', born:1950})\\n\",\n \"CREATE (BillPax:Person {name:'Bill Paxton', born:1955})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Jim Lovell']}]->(Apollo13),\\n\",\n \"(KevinB)-[:ACTED_IN {roles:['Jack Swigert']}]->(Apollo13),\\n\",\n \"(EdH)-[:ACTED_IN {roles:['Gene Kranz']}]->(Apollo13),\\n\",\n \"(BillPax)-[:ACTED_IN {roles:['Fred Haise']}]->(Apollo13),\\n\",\n \"(GaryS)-[:ACTED_IN {roles:['Ken Mattingly']}]->(Apollo13),\\n\",\n \"(RonH)-[:DIRECTED]->(Apollo13)\\n\",\n \"\\n\",\n \"CREATE (Twister:Movie {title:'Twister', released:1996, tagline:\\\"Don't Breathe. Don't Look Back.\\\"})\\n\",\n \"CREATE (PhilipH:Person {name:'Philip Seymour Hoffman', born:1967})\\n\",\n \"CREATE (JanB:Person {name:'Jan de Bont', born:1943})\\n\",\n \"CREATE\\n\",\n \"(BillPax)-[:ACTED_IN {roles:['Bill Harding']}]->(Twister),\\n\",\n \"(HelenH)-[:ACTED_IN {roles:['Dr. Jo Harding']}]->(Twister),\\n\",\n \"(ZachG)-[:ACTED_IN {roles:['Eddie']}]->(Twister),\\n\",\n \"(PhilipH)-[:ACTED_IN {roles:['Dustin \\\"Dusty\\\" Davis']}]->(Twister),\\n\",\n \"(JanB)-[:DIRECTED]->(Twister)\\n\",\n \"\\n\",\n \"CREATE (CastAway:Movie {title:'Cast Away', released:2000, tagline:'At the edge of the world, his journey begins.'})\\n\",\n \"CREATE (RobertZ:Person {name:'Robert Zemeckis', born:1951})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Chuck Noland']}]->(CastAway),\\n\",\n \"(HelenH)-[:ACTED_IN {roles:['Kelly Frears']}]->(CastAway),\\n\",\n \"(RobertZ)-[:DIRECTED]->(CastAway)\\n\",\n \"\\n\",\n \"CREATE (OneFlewOvertheCuckoosNest:Movie {title:\\\"One Flew Over the Cuckoo's Nest\\\", released:1975, tagline:\\\"If he's crazy, what does that make you?\\\"})\\n\",\n \"CREATE (MilosF:Person {name:'Milos Forman', born:1932})\\n\",\n \"CREATE\\n\",\n \"(JackN)-[:ACTED_IN {roles:['Randle McMurphy']}]->(OneFlewOvertheCuckoosNest),\\n\",\n \"(DannyD)-[:ACTED_IN {roles:['Martini']}]->(OneFlewOvertheCuckoosNest),\\n\",\n \"(MilosF)-[:DIRECTED]->(OneFlewOvertheCuckoosNest)\\n\",\n \"\\n\",\n \"CREATE (SomethingsGottaGive:Movie {title:\\\"Something's Gotta Give\\\", released:2003})\\n\",\n \"CREATE (DianeK:Person {name:'Diane Keaton', born:1946})\\n\",\n \"CREATE (NancyM:Person {name:'Nancy Meyers', born:1949})\\n\",\n \"CREATE\\n\",\n \"(JackN)-[:ACTED_IN {roles:['Harry Sanborn']}]->(SomethingsGottaGive),\\n\",\n \"(DianeK)-[:ACTED_IN {roles:['Erica Barry']}]->(SomethingsGottaGive),\\n\",\n \"(Keanu)-[:ACTED_IN {roles:['Julian Mercer']}]->(SomethingsGottaGive),\\n\",\n \"(NancyM)-[:DIRECTED]->(SomethingsGottaGive),\\n\",\n \"(NancyM)-[:PRODUCED]->(SomethingsGottaGive),\\n\",\n \"(NancyM)-[:WROTE]->(SomethingsGottaGive)\\n\",\n \"\\n\",\n \"CREATE (BicentennialMan:Movie {title:'Bicentennial Man', released:1999, tagline:\\\"One robot's 200 year journey to become an ordinary man.\\\"})\\n\",\n \"CREATE (ChrisC:Person {name:'Chris Columbus', born:1958})\\n\",\n \"CREATE\\n\",\n \"(Robin)-[:ACTED_IN {roles:['Andrew Marin']}]->(BicentennialMan),\\n\",\n \"(OliverP)-[:ACTED_IN {roles:['Rupert Burns']}]->(BicentennialMan),\\n\",\n \"(ChrisC)-[:DIRECTED]->(BicentennialMan)\\n\",\n \"\\n\",\n \"CREATE (CharlieWilsonsWar:Movie {title:\\\"Charlie Wilson's War\\\", released:2007, tagline:\\\"A stiff drink. A little mascara. A lot of nerve. Who said they couldn't bring down the Soviet empire.\\\"})\\n\",\n \"CREATE (JuliaR:Person {name:'Julia Roberts', born:1967})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Rep. Charlie Wilson']}]->(CharlieWilsonsWar),\\n\",\n \"(JuliaR)-[:ACTED_IN {roles:['Joanne Herring']}]->(CharlieWilsonsWar),\\n\",\n \"(PhilipH)-[:ACTED_IN {roles:['Gust Avrakotos']}]->(CharlieWilsonsWar),\\n\",\n \"(MikeN)-[:DIRECTED]->(CharlieWilsonsWar)\\n\",\n \"\\n\",\n \"CREATE (ThePolarExpress:Movie {title:'The Polar Express', released:2004, tagline:'This Holiday Season... Believe'})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Hero Boy', 'Father', 'Conductor', 'Hobo', 'Scrooge', 'Santa Claus']}]->(ThePolarExpress),\\n\",\n \"(RobertZ)-[:DIRECTED]->(ThePolarExpress)\\n\",\n \"\\n\",\n \"CREATE (ALeagueofTheirOwn:Movie {title:'A League of Their Own', released:1992, tagline:'Once in a lifetime you get a chance to do something different.'})\\n\",\n \"CREATE (Madonna:Person {name:'Madonna', born:1954})\\n\",\n \"CREATE (GeenaD:Person {name:'Geena Davis', born:1956})\\n\",\n \"CREATE (LoriP:Person {name:'Lori Petty', born:1963})\\n\",\n \"CREATE (PennyM:Person {name:'Penny Marshall', born:1943})\\n\",\n \"CREATE\\n\",\n \"(TomH)-[:ACTED_IN {roles:['Jimmy Dugan']}]->(ALeagueofTheirOwn),\\n\",\n \"(GeenaD)-[:ACTED_IN {roles:['Dottie Hinson']}]->(ALeagueofTheirOwn),\\n\",\n \"(LoriP)-[:ACTED_IN {roles:['Kit Keller']}]->(ALeagueofTheirOwn),\\n\",\n \"(RosieO)-[:ACTED_IN {roles:['Doris Murphy']}]->(ALeagueofTheirOwn),\\n\",\n \"(Madonna)-[:ACTED_IN {roles:['\\\"All the Way\\\" Mae Mordabito']}]->(ALeagueofTheirOwn),\\n\",\n \"(BillPax)-[:ACTED_IN {roles:['Bob Hinson']}]->(ALeagueofTheirOwn),\\n\",\n \"(PennyM)-[:DIRECTED]->(ALeagueofTheirOwn)\\n\",\n \"\\n\",\n \"CREATE (PaulBlythe:Person {name:'Paul Blythe'})\\n\",\n \"CREATE (AngelaScope:Person {name:'Angela Scope'})\\n\",\n \"CREATE (JessicaThompson:Person {name:'Jessica Thompson'})\\n\",\n \"CREATE (JamesThompson:Person {name:'James Thompson'})\\n\",\n \"\\n\",\n \"CREATE\\n\",\n \"(JamesThompson)-[:FOLLOWS]->(JessicaThompson),\\n\",\n \"(AngelaScope)-[:FOLLOWS]->(JessicaThompson),\\n\",\n \"(PaulBlythe)-[:FOLLOWS]->(AngelaScope)\\n\",\n \"\\n\",\n \"CREATE\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:'An amazing journey', rating:95}]->(CloudAtlas),\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:'Silly, but fun', rating:65}]->(TheReplacements),\\n\",\n \"(JamesThompson)-[:REVIEWED {summary:'The coolest football movie ever', rating:100}]->(TheReplacements),\\n\",\n \"(AngelaScope)-[:REVIEWED {summary:'Pretty funny at times', rating:62}]->(TheReplacements),\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:'Dark, but compelling', rating:85}]->(Unforgiven),\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:\\\"Slapstick redeemed only by the Robin Williams and Gene Hackman's stellar performances\\\", rating:45}]->(TheBirdcage),\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:'A solid romp', rating:68}]->(TheDaVinciCode),\\n\",\n \"(JamesThompson)-[:REVIEWED {summary:'Fun, but a little far fetched', rating:65}]->(TheDaVinciCode),\\n\",\n \"(JessicaThompson)-[:REVIEWED {summary:'You had me at Jerry', rating:92}]->(JerryMaguire)\\n\",\n \"\\n\",\n \"WITH TomH as a\\n\",\n \"MATCH (a)-[:ACTED_IN]->(m)<-[:DIRECTED]-(d) RETURN a,m,d LIMIT 10;\"\n ],\n \"metadata\": {\n \"id\": \"QFbjo1k24YEY\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"You can query the database via **cy2py** in this simple way\"\n ],\n \"metadata\": {\n \"id\": \"peqcEHj0b35T\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"CALL apoc.meta.graph()\"\n ],\n \"metadata\": {\n \"id\": \"BfFOTNkncMqp\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"As you can see the model is exactely how we expect!\"\n ],\n \"metadata\": {\n \"id\": \"sGu-zpk8nY5r\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# this step is MANDATORY for the exercises\\n\",\n \"from neo4j import GraphDatabase\\n\",\n \"neo4j_driver = GraphDatabase.driver(neo4j_url, auth=(neo4j_user, neo4j_password))\"\n ],\n \"metadata\": {\n \"id\": \"_zZF1guo58cc\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"c8bQe1b-7RY-\"\n },\n \"source\": [\n \"# Read data from Neo4j into Spark\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"The query above generates the following graph model:\\n\",\n \"\\n\"\n ],\n \"metadata\": {\n \"id\": \"ovoUnDmocaxK\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"B1LLHYf1CsPh\"\n },\n \"source\": [\n \"## Read nodes via `labels` option\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"omdSk6ShCqfA\"\n },\n \"outputs\": [],\n \"source\": [\n \"movies_df = (spark.read\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .option('labels', ':Movie')\\n\",\n \" .load())\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"RyglSgXnQcar\"\n },\n \"source\": [\n \"### Schema description\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"f9AaUINjPH4n\"\n },\n \"outputs\": [],\n \"source\": [\n \"movies_df.printSchema()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"The `movies_df` contains a set of fields, the first two (generally) are always:\\n\",\n \"\\n\",\n \"* `` which represents the internal Neo4j id\\n\",\n \"* `` which represents the list of labels attached to the node\\n\",\n \"\\n\",\n \"All other properties are taken from the node via schema resolution by using APOC or Cypher queries\"\n ],\n \"metadata\": {\n \"id\": \"jxLcYSkgZ1xf\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"omGjaw5QDgS-\"\n },\n \"outputs\": [],\n \"source\": [\n \"movies_df\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"7-KTRC5HD5sO\"\n },\n \"source\": [\n \"### Exercise\\n\",\n \"\\n\",\n \"Read all the `Person` nodes store them into a Python variable called `person_df` and then verify the results\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"ZhnsFC9KEsLp\"\n },\n \"outputs\": [],\n \"source\": [\n \"person_df = # write your spark code here\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"person_df = (spark.read\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .option('labels', ':Person')\\n\",\n \" .load())\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\\n\",\n \"\\n\"\n ],\n \"metadata\": {\n \"id\": \"O4WEzidAZBh-\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"D5_zXyweE-QM\"\n },\n \"outputs\": [],\n \"source\": [\n \"\\\"\\\"\\\"\\n\",\n \" This paragraph is for validating the code the you\\n\",\n \" wrote above, please execute it after you\\n\",\n \" created the person_df\\n\",\n \"\\\"\\\"\\\"\\n\",\n \"\\n\",\n \"assert person_df.count() == 133\\n\",\n \"assert person_df.schema.fieldNames() == ['', '', 'name', 'born']\\n\",\n \"assert person_df.collect()[0][\\\"\\\"] == ['Person']\\n\",\n \"print(\\\"All assertion are successfuly satisfied. Congrats you created your first DataFrame\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"m1hgGMLCRoZx\"\n },\n \"source\": [\n \"## Read relationships via `relationship` option\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"HgPockV0I5Q3\"\n },\n \"source\": [\n \"There are two way to transform relationships into DataFrame\\n\",\n \"\\n\",\n \"* having all the node and relationship data flattened into the DataFrame\\n\",\n \"* having all the node properties in maps and the relationship data as columns\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"m0DBqZLtKtvX\"\n },\n \"source\": [\n \"### DataFrame with flattened data\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"796cuMwXR2zi\"\n },\n \"outputs\": [],\n \"source\": [\n \"actedin_df = (spark.read\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .option('relationship', 'ACTED_IN')\\n\",\n \" .option('relationship.source.labels', ':Person')\\n\",\n \" .option('relationship.target.labels', ':Movie')\\n\",\n \" .load())\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Schema description\"\n ],\n \"metadata\": {\n \"id\": \"yzyviI5vXO4K\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"5uDWZqoySNGc\"\n },\n \"outputs\": [],\n \"source\": [\n \"actedin_df.printSchema()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"The `movies_df` contains a set of fields, the first two (generally) are always:\\n\",\n \"\\n\",\n \"* `` which represents the internal Neo4j relationship id\\n\",\n \"* `` which represents the relationship type\\n\",\n \"* `` which represents the internal Neo4j node id\\n\",\n \"* `` which represents the list of labels attached to the node\\n\",\n \"* `rel.*` which represents the properties attached to the relationship\\n\",\n \"* `source/target.*` which represents the properties attached to the node\\n\",\n \"\\n\",\n \"All other properties are taken from the node via schema resolution by using APOC or Cypher queries\"\n ],\n \"metadata\": {\n \"id\": \"2dB9DL7KZxrX\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"VPHDTL-IUX2X\"\n },\n \"outputs\": [],\n \"source\": [\n \"actedin_df\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"RoPVDptGKy_m\"\n },\n \"source\": [\n \"### DataFrame with nodes as map\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"8VxgDlBXIt_h\"\n },\n \"outputs\": [],\n \"source\": [\n \"actedin_map_df = (spark.read\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .option('relationship.nodes.map', True)\\n\",\n \" .option('relationship', 'ACTED_IN')\\n\",\n \" .option('relationship.source.labels', ':Person')\\n\",\n \" .option('relationship.target.labels', ':Movie')\\n\",\n \" .load())\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Schema description\"\n ],\n \"metadata\": {\n \"id\": \"nbGAYcd7YMOp\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"gF5-m4BbI2Ib\"\n },\n \"outputs\": [],\n \"source\": [\n \"actedin_map_df.printSchema()\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"The `movies_df` contains a set of fields, the first two (generally) are always:\\n\",\n \"\\n\",\n \"* `` which represents the internal Neo4j relationship id\\n\",\n \"* `` which represents the relationship type\\n\",\n \"* `` which represents a map with node values\\n\",\n \"* `rel.*` which represents the properties attached to the relationship\\n\",\n \"\\n\",\n \"All other properties are taken from the node via schema resolution by using APOC or Cypher queries\"\n ],\n \"metadata\": {\n \"id\": \"Zuu42SpfZ502\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"UUYUwE3CLA_r\"\n },\n \"outputs\": [],\n \"source\": [\n \"actedin_map_df\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"actedin_map_df.collect()[0][\\\"\\\"]\"\n ],\n \"metadata\": {\n \"id\": \"hHPq7neyYDSx\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Exercise\\n\",\n \"\\n\",\n \"Read all the `DIRECTED` relationships\"\n ],\n \"metadata\": {\n \"id\": \"Viop-9_thCbF\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"directed_df = # write your spark code here\"\n ],\n \"metadata\": {\n \"id\": \"j0tTsk59hhLh\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"directed_df = (spark.read\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .option('relationship', 'DIRECTED')\\n\",\n \" .option('relationship.source.labels', ':Person')\\n\",\n \" .option('relationship.target.labels', ':Movie')\\n\",\n \" .load())\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\"\n ],\n \"metadata\": {\n \"id\": \"VQYyYSMpj2lf\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"\\\"\\\"\\\"\\n\",\n \" This paragraph is for validating the code the you\\n\",\n \" wrote above, please execute it after you\\n\",\n \" created the directed_df\\n\",\n \"\\\"\\\"\\\"\\n\",\n \"\\n\",\n \"assert directed_df.count() == 44\\n\",\n \"assert directed_df.schema.fieldNames() == ['',\\n\",\n \" '',\\n\",\n \" '',\\n\",\n \" '',\\n\",\n \" 'source.name',\\n\",\n \" 'source.born',\\n\",\n \" '',\\n\",\n \" '',\\n\",\n \" 'target.title',\\n\",\n \" 'target.tagline',\\n\",\n \" 'target.released']\\n\",\n \"assert directed_df.collect()[0][\\\"\\\"] == 'DIRECTED'\\n\",\n \"print(\\\"All assertion are successfuly satisfied. Congrats you created your first relationship DataFrame\\\")\"\n ],\n \"metadata\": {\n \"id\": \"HRwaJ8PvhudP\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Read arbitrary data via Cypher query\"\n ],\n \"metadata\": {\n \"id\": \"hCpzW904dS2r\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"cypher_df = (spark.read\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .option('query', '''\\n\",\n \" // Extend Tom Hanks co-actors, to find co-co-actors who haven't worked with Tom Hanks\\n\",\n \" MATCH (tom:Person {name:\\\"Tom Hanks\\\"})-[:ACTED_IN]->(m)<-[:ACTED_IN]-(coActors),\\n\",\n \" (coActors)-[:ACTED_IN]->(m2)<-[:ACTED_IN]-(cocoActors)\\n\",\n \" WHERE NOT (tom)-[:ACTED_IN]->()<-[:ACTED_IN]-(cocoActors)\\n\",\n \" AND tom <> cocoActors\\n\",\n \" RETURN cocoActors.name AS Recommended, count(*) AS Strength\\n\",\n \" ORDER BY Strength DESC\\n\",\n \" ''')\\n\",\n \" .load())\"\n ],\n \"metadata\": {\n \"id\": \"hplBy0b_dhnb\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Schema description\"\n ],\n \"metadata\": {\n \"id\": \"tRZPA6xWeSCT\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"cypher_df.printSchema()\"\n ],\n \"metadata\": {\n \"id\": \"hU-JfgNNeL5f\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"cypher_df\"\n ],\n \"metadata\": {\n \"id\": \"8IcUQsileXQ7\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"// Just for debugging purposes let's check the same query directly from the database\\n\",\n \"MATCH (tom:Person {name:\\\"Tom Hanks\\\"})-[:ACTED_IN]->(m)<-[:ACTED_IN]-(coActors),\\n\",\n \" (coActors)-[:ACTED_IN]->(m2)<-[:ACTED_IN]-(cocoActors)\\n\",\n \"WHERE NOT (tom)-[:ACTED_IN]->()<-[:ACTED_IN]-(cocoActors)\\n\",\n \" AND tom <> cocoActors\\n\",\n \"RETURN cocoActors.name AS Recommended, count(*) AS Strength\\n\",\n \"ORDER BY Strength DESC\\n\",\n \"LIMIT 20\"\n ],\n \"metadata\": {\n \"id\": \"CWtNAeoN4O6S\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Exercise\\n\",\n \"\\n\",\n \"Return all the actors that have also directed a movie.\\n\",\n \"\\n\",\n \"The returned DataFrame must have 3 columns:\\n\",\n \"\\n\",\n \"* `name` the actor name\\n\",\n \"* `acted_in` a list of unique films (title) where he acted in\\n\",\n \"* `directed` a list of unique films (title) where he was a director\"\n ],\n \"metadata\": {\n \"id\": \"Hyq4KsKQegdE\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"your_cypher_df = # write your spark code here\"\n ],\n \"metadata\": {\n \"id\": \"0h1FFuYxej2f\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"your_cypher_df = (spark.read\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .option('query', '''\\n\",\n \" MATCH (p:Person)\\n\",\n \" MATCH (p)-[:ACTED_IN]->(m:Movie)\\n\",\n \" MATCH (p)-[:DIRECTED]->(m1:Movie)\\n\",\n \" RETURN p.name AS name, collect(m.title) AS acted_in, collect(m1.title) AS directed\\n\",\n \" ''')\\n\",\n \" .load())\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\"\n ],\n \"metadata\": {\n \"id\": \"CQncCY52lCxv\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"\\\"\\\"\\\"\\n\",\n \" This paragraph is for validating the code the you\\n\",\n \" wrote above, please execute it after you\\n\",\n \" created the your_cypher_df\\n\",\n \"\\\"\\\"\\\"\\n\",\n \"\\n\",\n \"assert your_cypher_df.count() == 5\\n\",\n \"assert your_cypher_df.schema.fieldNames() == ['name', 'acted_in', 'directed']\\n\",\n \"your_cypher_df_collect = your_cypher_df.collect()\\n\",\n \"assert frozenset(map(lambda row: row['name'], your_cypher_df_collect)) == frozenset(['Clint Eastwood',\\n\",\n \" 'Danny DeVito',\\n\",\n \" 'James Marshall',\\n\",\n \" 'Werner Herzog',\\n\",\n \" 'Tom Hanks'])\\n\",\n \"assert frozenset(map(lambda row: frozenset(row['acted_in']), your_cypher_df_collect)) == set([\\n\",\n \" frozenset([\\\"Apollo 13\\\", \\\"You've Got Mail\\\", \\\"A League of Their Own\\\", \\\"Joe Versus the Volcano\\\", \\\"That Thing You Do\\\", \\\"The Da Vinci Code\\\", \\\"Cloud Atlas\\\", \\\"Cast Away\\\", \\\"The Green Mile\\\", \\\"Sleepless in Seattle\\\", \\\"The Polar Express\\\", \\\"Charlie Wilson's War\\\"]),\\n\",\n \" frozenset([\\\"What Dreams May Come\\\"]),\\n\",\n \" frozenset([\\\"Unforgiven\\\"]),\\n\",\n \" frozenset([\\\"A Few Good Men\\\"]),\\n\",\n \" frozenset([\\\"Hoffa\\\", \\\"One Flew Over the Cuckoo's Nest\\\"])\\n\",\n \" ])\\n\",\n \"assert frozenset(map(lambda row: frozenset(row['directed']), your_cypher_df_collect)) == set([\\n\",\n \" frozenset([\\\"That Thing You Do\\\"]),\\n\",\n \" frozenset([\\\"RescueDawn\\\"]),\\n\",\n \" frozenset([\\\"Unforgiven\\\"]),\\n\",\n \" frozenset([\\\"V for Vendetta\\\", \\\"Ninja Assassin\\\"]),\\n\",\n \" frozenset([\\\"Hoffa\\\"])\\n\",\n \" ])\\n\",\n \"print(\\\"All assertion are successfuly satisfied. Congrats you created your first cypher dataframe\\\")\"\n ],\n \"metadata\": {\n \"id\": \"xG_7Wy-_go5V\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"hFpA11aK8ADf\"\n },\n \"source\": [\n \"# Write data from Spark to Neo4j\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## The graph model\\n\",\n \"\\n\",\n \"Our goal is to create this simple graph model\\n\",\n \"\\n\",\n \"\"\n ],\n \"metadata\": {\n \"id\": \"Mx84Qi1PcHF_\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"Trt-L_9pMQf1\"\n },\n \"source\": [\n \"### Download The Dataset\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"h4o07NpuJmaG\"\n },\n \"outputs\": [],\n \"source\": [\n \"!wget -q https://s3.amazonaws.com/dev.assets.neo4j.com/wp-content/uploads/desktop-csv-import.zip\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"The zip is composed of three files:\\n\",\n \"* products.csv: describes the products and has three columns (and no header)\\n\",\n \"* orders.csv: has three columns (with the header) and describe the order\\n\",\n \"* order-details.csv: is the \\\"join\\\" table between orders and products; it has three columns with header\"\n ],\n \"metadata\": {\n \"id\": \"KKfl_ZyhYYWj\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"nduIG7H_J0-A\"\n },\n \"outputs\": [],\n \"source\": [\n \"!unzip desktop-csv-import.zip\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"w5kaTPoEQNkT\"\n },\n \"source\": [\n \"### Explore the Dataset\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"L0TZgi_E1gAv\"\n },\n \"outputs\": [],\n \"source\": [\n \"products_df = (spark.read\\n\",\n \" .format('csv')\\n\",\n \" .option('inferSchema', True)\\n\",\n \" .option('path', '/content/desktop-csv-import/products.csv')\\n\",\n \" .load())\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"0tptDEUn2WO6\"\n },\n \"outputs\": [],\n \"source\": [\n \"products_df.printSchema()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"ZSW40PxjKvgf\"\n },\n \"outputs\": [],\n \"source\": [\n \"products_df\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"As you can see in the schema, colums have no name, just a generic `_c` prefix concatenated with an index.\\n\",\n \"The three columns describe:\\n\",\n \"* `_c0` is the `id` of the product\\n\",\n \"* `_c1` is the `name`\\n\",\n \"* `_c2` is the `price`\\n\",\n \"\\n\",\n \"Let's rename these columns!\"\n ],\n \"metadata\": {\n \"id\": \"VrKERh4uXu8J\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"7xabRYlXQZl4\"\n },\n \"outputs\": [],\n \"source\": [\n \"products_df = (products_df.withColumnRenamed('_c0', 'id')\\n\",\n \" .withColumnRenamed('_c1', 'name')\\n\",\n \" .withColumnRenamed('_c2', 'price'))\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"w08HTcIeQuNn\"\n },\n \"outputs\": [],\n \"source\": [\n \"products_df.printSchema()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"XCkyYIyzQv0X\"\n },\n \"outputs\": [],\n \"source\": [\n \"products_df\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Write nodes via `label` option\"\n ],\n \"metadata\": {\n \"id\": \"XL9oBe0-m680\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"Oy4eoeAMRWxc\"\n },\n \"outputs\": [],\n \"source\": [\n \"(products_df.write\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .mode('append')\\n\",\n \" .option('labels', ':Product')\\n\",\n \" .save())\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Let's check if the nodes are in the database!\"\n ],\n \"metadata\": {\n \"id\": \"MLxewMeFsQj3\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH (n:Product)\\n\",\n \"RETURN n\\n\",\n \"LIMIT 10\"\n ],\n \"metadata\": {\n \"id\": \"2yoBPvmmsUqt\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Now just to be sure that we loaded all the nodes into Neo4j we'll count the dataframe and the nodes inside the database\"\n ],\n \"metadata\": {\n \"id\": \"10PLvdmZ0tZT\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"products_df.count()\"\n ],\n \"metadata\": {\n \"id\": \"AmwadsK702Sl\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH (n:Product)\\n\",\n \"RETURN count(n)\"\n ],\n \"metadata\": {\n \"id\": \"J8e9jZPG06DL\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"If the two counts are equal, all the data has been properly imported.\"\n ],\n \"metadata\": {\n \"id\": \"T8XAaw3K1Mga\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Create Constraints\\n\",\n \"\\n\",\n \"Oh but wait, we forgot to create constraints!!! if we go into the Neo4j browser and excute the following query:\\n\",\n \"\\n\",\n \"```cypher\\n\",\n \"show constraints\\n\",\n \"```\\n\",\n \"\\n\",\n \"We should get the constraints of the movie database, but not one for `Product`.\\n\",\n \"\\n\",\n \"So please create the constaints for the node `Product`:\\n\",\n \"\\n\",\n \"```cypher\\n\",\n \"CREATE CONSTRAINT product_id FOR (p:Product) REQUIRE p.id IS UNIQUE;\\n\",\n \"```\\n\",\n \"\\n\",\n \"But if you want, you can also delegate the Spark connector to perform optimizations pre-processing by usign the option `schema.optimization.type` which can assume three values:\\n\",\n \"\\n\",\n \"* `INDEX`: it creates only indexes on provided nodes.\\n\",\n \"* `NODE_CONSTRAINTS`: it creates only indexes on provided nodes.\\n\",\n \"\\n\",\n \"So let's create the `Order` node with by let the connector creating the constraints for you\"\n ],\n \"metadata\": {\n \"id\": \"jFL_tQk0tsni\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"// Check the constraints\\n\",\n \"SHOW CONSTRAINTS\"\n ],\n \"metadata\": {\n \"id\": \"T3PUsfIvsi23\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"// Create the constraint for Product node\\n\",\n \"CREATE CONSTRAINT product_id IF NOT EXISTS FOR (p:Product) REQUIRE p.id IS UNIQUE;\"\n ],\n \"metadata\": {\n \"id\": \"iMF68OU20XhE\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"// Check (again) the constraints\\n\",\n \"SHOW CONSTRAINTS\"\n ],\n \"metadata\": {\n \"id\": \"w_wSMYvz0fz_\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"As you can see now we have the `product_id` constraint in the result list\"\n ],\n \"metadata\": {\n \"id\": \"2JwYLMva1VnN\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"orders_df = (spark.read\\n\",\n \" .format('csv')\\n\",\n \" .option('inferSchema', True)\\n\",\n \" .option('header', True)\\n\",\n \" .option('path', '/content/desktop-csv-import/orders.csv')\\n\",\n \" .load())\"\n ],\n \"metadata\": {\n \"id\": \"m5ge2R_Ggd3K\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"orders_df.printSchema()\"\n ],\n \"metadata\": {\n \"id\": \"7UGwHtuJwFU4\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"orders_df\"\n ],\n \"metadata\": {\n \"id\": \"uGmUP5ZMwJkm\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# we cast orderDate to timestamp in order to have it converted properly into Neo4j\\n\",\n \"orders_df = orders_df.selectExpr('orderID AS id', 'CAST(orderDate AS TIMESTAMP) AS date', 'shipCountry')\"\n ],\n \"metadata\": {\n \"id\": \"InwYglcUwXNy\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"orders_df.printSchema()\"\n ],\n \"metadata\": {\n \"id\": \"kSJmW81cw_ES\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"orders_df\"\n ],\n \"metadata\": {\n \"id\": \"7VN5RAizxSr2\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"(orders_df.write\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .mode('overwrite')\\n\",\n \" .option('labels', ':Order')\\n\",\n \" .option('schema.optimization.type', 'NODE_CONSTRAINTS')\\n\",\n \" # this is necessary in order to specify what is the constraint field\\n\",\n \" .option('node.keys', 'id')\\n\",\n \" .save())\"\n ],\n \"metadata\": {\n \"id\": \"0rwKB9V-xTzu\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Now let's check if the connector has created the constraint for us\"\n ],\n \"metadata\": {\n \"id\": \"A_T2beEJx3Ho\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"SHOW CONSTRAINTS\"\n ],\n \"metadata\": {\n \"id\": \"hDxdKyFY1sDT\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"As you can see the we have the constraint `spark_NODE_CONSTRAINTS_Order_id` that has been create by the Spark connector itself.\\n\",\n \"\\n\",\n \"Now just because we're courious let's check if the data has been propertly loaded.\\n\",\n \"\\n\",\n \"The first thing to check is if the count of the Dataframe and the nodes in Neo4j matches.\"\n ],\n \"metadata\": {\n \"id\": \"H5Bne-Mq1vsO\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"orders_df.count()\"\n ],\n \"metadata\": {\n \"id\": \"mANrH-Zt2ShO\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH (o:Order)\\n\",\n \"RETURN count(o)\"\n ],\n \"metadata\": {\n \"id\": \"vzCCVYAK2V0X\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Now we want to check if the data has been loaded with the proper data type, in particular we created a new column `date` by casting `orderDate` to `TIMESTAMP`.\"\n ],\n \"metadata\": {\n \"id\": \"X8BBxCQg2dFM\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH (o:Order)\\n\",\n \"RETURN apoc.meta.cypher.type(o.date), count(o)\"\n ],\n \"metadata\": {\n \"id\": \"VN7XMI192xAP\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"So all the `date` values have the same type.\"\n ],\n \"metadata\": {\n \"id\": \"PpXyKtJF3Lk4\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Exercise\\n\",\n \"\\n\",\n \"Given the `football_teams_df` and `football_player_df` below please:\\n\",\n \"* for `football_teams_df` insert it as nodes with label `:FootballTeam` in Neo4j.\\n\",\n \"* for `football_player_df` insert it as nodes with label `:FootballPlayer` in Neo4j.\\n\",\n \"\\n\",\n \"Create for both of them constraints via the schema optimization feature:\\n\",\n \"* for `football_teams_df` the key must be the property `id`\\n\",\n \"* for `football_player_df` the key must be the property `name`\"\n ],\n \"metadata\": {\n \"id\": \"Zi4Dl2LqmVmN\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"football_teams_df = spark.createDataFrame([{'id': 1, 'name': 'AC Milan'}, {'id': 2, 'name': 'FC Internazionale'}])\\n\",\n \"football_player_df = spark.createDataFrame([\\n\",\n \" {'name': 'Zlatan Ibrahimovic'},\\n\",\n \" {'name': 'Sandro Tonali'},\\n\",\n \" {'name': 'Nicolò Barella'},\\n\",\n \" {'name': 'Marcelo Brozovic'}])\"\n ],\n \"metadata\": {\n \"id\": \"21tFDtgAmVON\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# write your spark code that persist football_teams_df and football_player_df here\"\n ],\n \"metadata\": {\n \"id\": \"utpvz-fI6blD\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"# write the teams\\n\",\n \"(football_teams_df.write\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .mode('overwrite')\\n\",\n \" .option('labels', ':FootballTeam')\\n\",\n \" .option('schema.optimization.type', 'NODE_CONSTRAINTS')\\n\",\n \" .option('node.keys', 'id')\\n\",\n \" .save())\\n\",\n \"# write the players\\n\",\n \"(football_player_df.write\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .mode('overwrite')\\n\",\n \" .option('labels', ':FootballPlayer')\\n\",\n \" .option('schema.optimization.type', 'NODE_CONSTRAINTS')\\n\",\n \" .option('node.keys', 'name')\\n\",\n \" .save())\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\"\n ],\n \"metadata\": {\n \"id\": \"xeVxPe7PmEy8\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"\\\"\\\"\\\"\\n\",\n \" This paragraph is for validating the code the you\\n\",\n \" wrote above, please execute it after you\\n\",\n \" persisted football_teams_df and\\n\",\n \" football_player_df in Neo4j as nodes\\n\",\n \"\\\"\\\"\\\"\\n\",\n \"\\n\",\n \"with neo4j_driver.session() as session:\\n\",\n \" # count football players\\n\",\n \" football_players = session.read_transaction(lambda tx: (tx.run('''\\n\",\n \" MATCH (p:FootballPlayer)\\n\",\n \" WHERE p.name IN ['Zlatan Ibrahimovic', 'Sandro Tonali',\\n\",\n \" 'Nicolò Barella', 'Marcelo Brozovic']\\n\",\n \" RETURN count(p) AS count\\n\",\n \" ''').single()['count']))\\n\",\n \" assert football_players == 4\\n\",\n \"\\n\",\n \" # count football teams\\n\",\n \" football_teams = session.read_transaction(lambda tx: (tx.run('''\\n\",\n \" MATCH (p:FootballTeam)\\n\",\n \" WHERE p.name IN ['AC Milan', 'FC Internazionale']\\n\",\n \" RETURN count(p) AS count\\n\",\n \" ''').single()['count']))\\n\",\n \" assert football_teams == 2\\n\",\n \"\\n\",\n \" # count constraints\\n\",\n \" football_constraints = session.read_transaction(lambda tx: (tx.run('''\\n\",\n \" SHOW CONSTRAINTS YIELD name\\n\",\n \" WHERE name IN ['spark_NODE_CONSTRAINTS_FootballPlayer_name', 'spark_NODE_CONSTRAINTS_FootballTeam_id']\\n\",\n \" RETURN count(*) AS count\\n\",\n \" ''').single()['count']))\\n\",\n \" assert football_constraints == 2\\n\",\n \"\\n\",\n \"print(\\\"All assertion are successfuly satisfied. Congrats you saved your first Node DataFrame into Neo4j!\\\")\"\n ],\n \"metadata\": {\n \"id\": \"a5zQCEyK6h5f\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Write relationships via `relationship` option\"\n ],\n \"metadata\": {\n \"id\": \"QltYkhIuy2Kc\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"order_details_df = (spark.read\\n\",\n \" .format('csv')\\n\",\n \" .option('inferSchema', True)\\n\",\n \" .option('header', True)\\n\",\n \" .option('path', '/content/desktop-csv-import/order-details.csv')\\n\",\n \" .load())\"\n ],\n \"metadata\": {\n \"id\": \"y3U_X0b7x2UN\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"order_details_df.printSchema()\"\n ],\n \"metadata\": {\n \"id\": \"zFqKW-j-zRyk\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"order_details_df\"\n ],\n \"metadata\": {\n \"id\": \"5s5f3W984Jc0\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Please remember that this is the pattern that we want to ingest:\\n\",\n \"\\n\",\n \"\\n\",\n \"\"\n ],\n \"metadata\": {\n \"id\": \"aZWSOtNK2qEw\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"(order_details_df.write\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .mode('overwrite')\\n\",\n \" .option('relationship', 'CONTAINS')\\n\",\n \" .option('relationship.save.strategy', 'keys')\\n\",\n \" .option('relationship.source.labels', ':Product')\\n\",\n \" .option('relationship.source.save.mode', 'Match')\\n\",\n \" .option('relationship.source.node.keys', 'productID:id')\\n\",\n \" .option('relationship.target.labels', ':Order')\\n\",\n \" .option('relationship.target.save.mode', 'Match')\\n\",\n \" .option('relationship.target.node.keys', 'orderID:id')\\n\",\n \" .option('relationship.properties', 'quantity:quantityOrdered')\\n\",\n \" .save())\"\n ],\n \"metadata\": {\n \"id\": \"rFo3KWA90rmZ\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Now let's check the count for both Dataframe and relationships in Neo4j\"\n ],\n \"metadata\": {\n \"id\": \"1PaC36iZ3bNf\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"order_details_df.count()\"\n ],\n \"metadata\": {\n \"id\": \"OUZ5FYHP3qvj\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH (p:Product)-[r:CONTAINS]->(o:Order)\\n\",\n \"RETURN count(r)\"\n ],\n \"metadata\": {\n \"id\": \"ex8o2pSo34cI\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Exercise\\n\",\n \"\\n\",\n \"Given the `team_player_df` create a relationship between `:FootballPlayer` and `:FootballTeam` of type `PLAYS_FOR`:\\n\",\n \"\\n\",\n \"```cypher\\n\",\n \"(:FootballPlayer)-[:PLAYS_FOR]->(:FootballTeam)\\n\",\n \"```\"\n ],\n \"metadata\": {\n \"id\": \"lTBo369687lC\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"team_player_df = spark.createDataFrame([\\n\",\n \" {'id': 1, 'football_player': 'Zlatan Ibrahimovic'},\\n\",\n \" {'id': 1, 'football_player': 'Sandro Tonali'},\\n\",\n \" {'id': 2, 'football_player': 'Nicolò Barella'},\\n\",\n \" {'id': 2, 'football_player': 'Marcelo Brozovic'}])\"\n ],\n \"metadata\": {\n \"id\": \"gi9kB0l49f8H\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# write your spark code that persist team_player_df here\"\n ],\n \"metadata\": {\n \"id\": \"i7ZjASDx_Kiy\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"(team_player_df.write\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .mode('overwrite')\\n\",\n \" .option('relationship', 'PLAYS_FOR')\\n\",\n \" .option('relationship.save.strategy', 'keys')\\n\",\n \" .option('relationship.source.labels', ':FootballPlayer')\\n\",\n \" .option('relationship.source.save.mode', 'Match')\\n\",\n \" .option('relationship.source.node.keys', 'football_player:name')\\n\",\n \" .option('relationship.target.labels', ':FootballTeam')\\n\",\n \" .option('relationship.target.save.mode', 'Match')\\n\",\n \" .option('relationship.target.node.keys', 'id')\\n\",\n \" .save())\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\"\n ],\n \"metadata\": {\n \"id\": \"oNRJevUSm0Wi\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"\\\"\\\"\\\"\\n\",\n \" This paragraph is for validating the code the you\\n\",\n \" wrote above, please execute it after you\\n\",\n \" persisted team_player_df as relationships\\n\",\n \"\\\"\\\"\\\"\\n\",\n \"\\n\",\n \"with neo4j_driver.session() as session:\\n\",\n \" # count relationships\\n\",\n \" def count_relationships(tx):\\n\",\n \" result = tx.run('''\\n\",\n \" MATCH (p:FootballPlayer)-[:PLAYS_FOR]->(t:FootballTeam)\\n\",\n \" RETURN t.name AS team, collect(p.name) AS players\\n\",\n \" ORDER by team\\n\",\n \" ''')\\n\",\n \" return [{'team': record['team'], 'players': set(record['players'])} for record in result]\\n\",\n \"\\n\",\n \" actual = session.read_transaction(count_relationships)\\n\",\n \" expected = [\\n\",\n \" {'team': 'AC Milan', 'players': frozenset(['Zlatan Ibrahimovic', 'Sandro Tonali'])},\\n\",\n \" {'team': 'FC Internazionale', 'players': frozenset(['Nicolò Barella', 'Marcelo Brozovic'])}\\n\",\n \" ]\\n\",\n \" assert actual == expected\\n\",\n \"\\n\",\n \"print(\\\"All assertion are successfuly satisfied. Congrats you saved your first Relationship DataFrame into Neo4j!\\\")\"\n ],\n \"metadata\": {\n \"id\": \"LDYbmoUx_Owb\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Write custom graphs via Cypher Query\\n\",\n \"\\n\",\n \"Now let's consider that two actors created an order and bought several products, and we want to add information in our database.\"\n ],\n \"metadata\": {\n \"id\": \"vBq9NCWlZkHw\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"actor_orders = [\\n\",\n \" {'actor_name': 'Cuba Gooding Jr.', 'order_id': 1, 'products': [11, 42, 72], 'quantities': [1, 2, 3], 'order_date': '2022-06-07 00:00:00'},\\n\",\n \" {'actor_name': 'Tom Hanks', 'order_id': 2, 'products': [24, 55, 75], 'quantities': [3, 2, 1], 'order_date': '2022-06-06 00:00:00'}\\n\",\n \"]\\n\",\n \"\\n\",\n \"actor_orders_df = spark.createDataFrame(actor_orders)\"\n ],\n \"metadata\": {\n \"id\": \"3y_yEOouaHxe\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"actor_orders_df.printSchema()\"\n ],\n \"metadata\": {\n \"id\": \"hYo7nlgvbfdm\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"actor_orders_df\"\n ],\n \"metadata\": {\n \"id\": \"fOkW4w2lbhZP\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"In this case please go into Neo4j and create the following constraint:\\n\",\n \"\\n\",\n \"```cypher\\n\",\n \"CREATE CONSTRAINT person_name FOR (p:Person) REQUIRE p.name is UNIQUE;\\n\",\n \"```\"\n ],\n \"metadata\": {\n \"id\": \"q0wAIB5l7qp9\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"// if you didn't before create the constraint on Person.name\\n\",\n \"CREATE CONSTRAINT person_name IF NOT EXISTS FOR (p:Person) REQUIRE p.name is UNIQUE;\"\n ],\n \"metadata\": {\n \"id\": \"-_Lxm_zV4qw2\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"(actor_orders_df.write\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .mode('overwrite')\\n\",\n \" .option('query', '''\\n\",\n \" MATCH (person:Person {name: event.actor_name})\\n\",\n \" MERGE (order:Order {id: event.order_id, date: datetime(replace(event.order_date, ' ', 'T'))})\\n\",\n \" MERGE (person)-[:CREATED]->(order)\\n\",\n \" WITH event, order\\n\",\n \" UNWIND range(0, size(event.products) - 1) AS index\\n\",\n \" MATCH (product:Product {id: event.products[index]})\\n\",\n \" MERGE (product)-[:CONTAINS{quantityOrdered: event.quantities[index]}]->(order)\\n\",\n \" ''')\\n\",\n \" .save())\"\n ],\n \"metadata\": {\n \"id\": \"cm3RQyLbbjue\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"What we expect now is that for the two actors there are two orders one per each, then each order contains three products.\"\n ],\n \"metadata\": {\n \"id\": \"61LUSN4F6pQq\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH (a:Person)-[:CREATED]->(o:Order)<-[c:CONTAINS]-(p:Product)\\n\",\n \"WHERE a.name IN ['Cuba Gooding Jr.', 'Tom Hanks']\\n\",\n \"RETURN a.name, o.id, o.date, p.name, c.quantityOrdered\"\n ],\n \"metadata\": {\n \"id\": \"8fVMQiTf61mN\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Exercise\\n\",\n \"\\n\",\n \"Given `neo4j_resources_df` build a small Knowledge Graph in Neo4j with the following structure:\\n\",\n \"\\n\",\n \"```cypher\\n\",\n \"(:Author{name})-[:CREATED]->(:Resource{name})-[:HAS_TAG]->(:Tag{name})\\n\",\n \"```\"\n ],\n \"metadata\": {\n \"id\": \"pn7IM8me9R3I\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"neo4j_resources_df = spark.createDataFrame([\\n\",\n \" {'author': 'LARUS Business Automation', 'resource': 'Galileo.XAI', 'tags': ['Graph Machine Learning', 'Neo4j', 'Explainable AI', 'Artificial Intelligence']},\\n\",\n \" {'author': 'Neo4j', 'resource': 'Graph Data Science Library', 'tags': ['Graph Machine Learning', 'Algorithms']},\\n\",\n \" {'author': 'Michael Hunger', 'resource': 'APOC', 'tags': ['Graph Data Integration', 'Graph Algorithms']}\\n\",\n \"])\"\n ],\n \"metadata\": {\n \"id\": \"_wmoLl8d9RVz\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"neo4j_resources_df\"\n ],\n \"metadata\": {\n \"id\": \"NkG1jCynLXgJ\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# write your spark code that persist neo4j_resources_df here\"\n ],\n \"metadata\": {\n \"id\": \"8EQmY-qhsbi1\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"(neo4j_resources_df.write\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .mode('overwrite')\\n\",\n \" .option('query', '''\\n\",\n \" MERGE (a:Author {name: event.author})\\n\",\n \" MERGE (r:Resource {name: event.resource})\\n\",\n \" MERGE (a)-[:CREATED]->(r)\\n\",\n \" WITH a, r, event\\n\",\n \" UNWIND event.tags AS tag\\n\",\n \" MERGE (t:Tag{name: tag})\\n\",\n \" MERGE (r)-[:HAS_TAG]->(t)\\n\",\n \" ''')\\n\",\n \" .save())\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\"\n ],\n \"metadata\": {\n \"id\": \"KCfw_saanywo\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"\\\"\\\"\\\"\\n\",\n \" This paragraph is for validating the code the you\\n\",\n \" wrote above, please execute it after you\\n\",\n \" persisted neo4j_resources_df as Cypher query\\n\",\n \"\\\"\\\"\\\"\\n\",\n \"\\n\",\n \"with neo4j_driver.session() as session:\\n\",\n \" # count relationships\\n\",\n \" def check_graph_consistency(tx):\\n\",\n \" result = tx.run('''\\n\",\n \" MATCH (a:Author)-[:CREATED]->(r:Resource)-[:HAS_TAG]->(t:Tag)\\n\",\n \" RETURN a.name AS author, r.name AS resource, collect(t.name) AS tags\\n\",\n \" ORDER By author\\n\",\n \" ''')\\n\",\n \" return [{'author': record['author'], 'resource': record['resource'], 'tags': set(record['tags'])} for record in result]\\n\",\n \"\\n\",\n \" actual = session.read_transaction(check_graph_consistency)\\n\",\n \" expected = [\\n\",\n \" {'author': 'LARUS Business Automation', 'resource': 'Galileo.XAI', 'tags': frozenset(['Graph Machine Learning', 'Neo4j', 'Explainable AI', 'Artificial Intelligence'])},\\n\",\n \" {'author': 'Michael Hunger', 'resource': 'APOC', 'tags': frozenset(['Graph Data Integration', 'Graph Algorithms'])},\\n\",\n \" {'author': 'Neo4j', 'resource': 'Graph Data Science Library', 'tags': frozenset(['Graph Machine Learning', 'Algorithms'])}\\n\",\n \" ]\\n\",\n \" assert actual == expected\\n\",\n \"\\n\",\n \"print(\\\"All assertion are successfuly satisfied. Congrats you saved your first Knowledge Graph DataFrame into Neo4j!\\\")\"\n ],\n \"metadata\": {\n \"id\": \"LwqbSsEcsgDi\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [],\n \"metadata\": {\n \"id\": \"_LbqufNZMj6-\"\n },\n \"execution_count\": null,\n \"outputs\": []\n }\n ],\n \"metadata\": {\n \"colab\": {\n \"provenance\": [],\n \"toc_visible\": true\n },\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"name\": \"python\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 0\n}" }, { "path": "examples/neo4j_data_science.ipynb", "content": "{\n \"nbformat\": 4,\n \"nbformat_minor\": 0,\n \"metadata\": {\n \"colab\": {\n \"provenance\": [],\n \"toc_visible\": true\n },\n \"kernelspec\": {\n \"name\": \"python3\",\n \"display_name\": \"Python 3\"\n },\n \"language_info\": {\n \"name\": \"python\"\n }\n },\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Open this notebook in Google Colab \\n\",\n \" \\\"Open\\n\",\n \"\"\n ],\n \"metadata\": {\n \"id\": \"ciNaixnkx1vj\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"# Example of a Simple data science workflow with Neo4j and Spark\"\n ],\n \"metadata\": {\n \"id\": \"zADiJjnuVfq2\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"This notebook contains a set of examples that explains how the Neo4j Spark connector can fit in you Data Scinece workflow, how you can combine Spark Neo4j and the Graph Data Science library to extract insights from your data and mostly important it allows you to test your knowledge with a set of exercises after each section.\\n\",\n \"\\n\",\n \"If you have any questions or problems feel free to write a post in the [Neo4j community forum](https://community.neo4j.com/) or in [Discord](https://discord.com/invite/neo4j).\\n\",\n \"\\n\",\n \"If you want more exercises feel free to open an issue in the [GitHub repository](https://github.com/neo4j/neo4j-spark-connector).\\n\",\n \"\\n\",\n \"Enjoy!\"\n ],\n \"metadata\": {\n \"id\": \"nLucMn17V0YK\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"# Notes about this notebook\\n\",\n \"\\n\",\n \"This code contains a simple data science workflow that combines Neo4j's Graph Data Science Library with the Neo4j Connector for Apache Spark.\\n\",\n \"\\n\",\n \"Going forward you'll find code examples in:\\n\",\n \"\\n\",\n \"* PySpark\\n\",\n \"* PySpark Pandas\\n\",\n \"\\n\",\n \"You can choose to navigate by using one of them, or both, but we suggest you do one at time to ensure you understand the APIs.\"\n ],\n \"metadata\": {\n \"id\": \"pWWY8190RB98\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"# Create the sandbox instance\\n\",\n \"\\n\",\n \"You can easily spin-up a Neo4j sandbox by click [here](https://sandbox.neo4j.com/?usecase=fraud-detection)\\n\",\n \"\\n\",\n \"After that you'll be redirect in a webpage like this:\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"Please click in the **Connection details tab** and copy your connection parameters into the Python variables below\"\n ],\n \"metadata\": {\n \"id\": \"3hCQBmBKVaHm\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"ttxf62TPVP-w\"\n },\n \"outputs\": [],\n \"source\": [\n \"neo4j_url = \\\"\\\" # put your neo4j url here\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"neo4j_user = \\\"neo4j\\\" # put your neo4j user here\"\n ],\n \"metadata\": {\n \"id\": \"-lPr1hfIGtfL\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"neo4j_password = \\\"\\\" # put your neo4j password here\"\n ],\n \"metadata\": {\n \"id\": \"yoI29jjvGvlX\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Configure the Spark Environment\"\n ],\n \"metadata\": {\n \"id\": \"Capd99x5G2rm\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"spark_version = '3.3.4'\"\n ],\n \"metadata\": {\n \"id\": \"OiHMiko1-Qf7\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"!apt-get install openjdk-17-jdk-headless -qq > /dev/null\"\n ],\n \"metadata\": {\n \"id\": \"qdjzLBDzGx5l\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"!wget -q https://dlcdn.apache.org/spark/spark-$spark_version/spark-$spark_version-bin-hadoop3.tgz\"\n ],\n \"metadata\": {\n \"id\": \"7JT9OKhzG7Lq\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"A3gsnSHl0F99\"\n },\n \"outputs\": [],\n \"source\": [\n \"!tar xf spark-$spark_version-bin-hadoop3.tgz\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"hSBQWKs90vSx\"\n },\n \"outputs\": [],\n \"source\": [\n \"!pip install -q findspark\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"tnW0a1Gj080k\"\n },\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"os.environ[\\\"JAVA_HOME\\\"] = \\\"/usr/lib/jvm/java-17-openjdk-amd64\\\"\\n\",\n \"os.environ[\\\"SPARK_HOME\\\"] = f\\\"/content/spark-{spark_version}-bin-hadoop3\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"dlUBSezK1DpZ\"\n },\n \"outputs\": [],\n \"source\": [\n \"import findspark\\n\",\n \"findspark.init()\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"dOUJ-W871Tur\"\n },\n \"outputs\": [],\n \"source\": [\n \"from pyspark.sql import SparkSession\\n\",\n \"spark = (SparkSession.builder\\n\",\n \" .master('local[*]')\\n\",\n \" .appName('Data science workflow with Neo4j and Spark')\\n\",\n \" .config('spark.ui.port', '4050')\\n\",\n \" # Just to show dataframes as tables\\n\",\n \" #.config('spark.sql.repl.eagerEval.enabled', False)\\n\",\n \" .config('spark.jars.packages', 'org.neo4j:neo4j-connector-apache-spark_2.12:5.1.0_for_spark_3')\\n\",\n \" # As we're using always the same database instance we'll\\n\",\n \" # define them as global variables\\n\",\n \" # so we don't need to repeat them each time\\n\",\n \" .config(\\\"neo4j.url\\\", neo4j_url)\\n\",\n \" .config(\\\"neo4j.authentication.type\\\", \\\"basic\\\")\\n\",\n \" .config(\\\"neo4j.authentication.basic.username\\\", neo4j_user)\\n\",\n \" .config(\\\"neo4j.authentication.basic.password\\\", neo4j_password)\\n\",\n \" .getOrCreate())\\n\",\n \"spark\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# import utility functions that we'll use in the notebook\\n\",\n \"from pyspark.sql.types import *\\n\",\n \"from pyspark.sql.functions import *\"\n ],\n \"metadata\": {\n \"id\": \"pghCcGnJWcZQ\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Import PySpark Pandas\\n\",\n \"\\n\",\n \"Pandas API on Apache Spark (PySpark) enables data scientists and data engineers to run their existing Pandas code on Spark. Prior to this API, you had to do a significant code rewrite from Pandas DataFrame to PySpark DataFrame which is time-consuming and error-prone.\\n\",\n \"\\n\",\n \"In this notebook we'll use both PySpark Dataframes and and PySpark Pandas.\\n\",\n \"\\n\",\n \"The only thing that we need to do is to import the library using the statement below.\"\n ],\n \"metadata\": {\n \"id\": \"klQ2Ah6CFBV1\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"import pyspark.pandas as ps\"\n ],\n \"metadata\": {\n \"id\": \"lDkBcHySCBT0\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"\\n\",\n \"## Exercises prerequisite\\n\",\n \"\\n\",\n \"In this notebook we and going to test your knowledge. Some of the exercises require the Neo4j Python driver to check if the exercises are being solved correctly.\\n\",\n \"\\n\",\n \"*Neo4j Python Driver is required only for verifying the exercises when you persist data from Spark to Neo4j*\\n\",\n \"\\n\",\n \"**It's not required by the Spark connector!!!**\\n\",\n \"\\n\",\n \"We'll use [Cy2Py](https://github.com/conker84/cy2py), a Jupyter extension that easily allows you to connect to Neo4j and visualize data from Jupyter notebooks.\\n\",\n \"For a detailed instruction about how to use it please dive into [this example](https://github.com/conker84/cy2py/blob/main/examples/Neo4j_Crime_Investigation_Dataset.ipynb)\"\n ],\n \"metadata\": {\n \"id\": \"b6_YNZnZ5GdT\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"!pip install -q cy2py\"\n ],\n \"metadata\": {\n \"id\": \"f5ZZJylo5Bbz\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"let's load the extension\"\n ],\n \"metadata\": {\n \"id\": \"uKYEPEgOcG2b\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%load_ext cy2py\"\n ],\n \"metadata\": {\n \"id\": \"38EeXF6icKOK\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"You can query the database via **cy2py** in this simple way\"\n ],\n \"metadata\": {\n \"id\": \"peqcEHj0b35T\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# define the colors for the nodes\\n\",\n \"colors = {\\n\",\n \" ':Client': '#D18711',\\n\",\n \" ':Bank': '#0541B2',\\n\",\n \" ':Merchant': '#9E14AA',\\n\",\n \" ':Mule': '#6113A3',\\n\",\n \" ':CashIn': '#328918',\\n\",\n \" ':CashOut': '#C1A23D',\\n\",\n \" ':Debit': '#A32727',\\n\",\n \" ':Payment': '#3B80C4',\\n\",\n \" ':Transfer': '#088472',\\n\",\n \" ':Transaction': '#D10B4F',\\n\",\n \" ':Email': '#EA5D1E',\\n\",\n \" ':SSN': '#707070',\\n\",\n \" ':Phone': '#4B4444',\\n\",\n \"}\"\n ],\n \"metadata\": {\n \"id\": \"dw2P-XpfLCJY\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher -u $neo4j_url -us $neo4j_user -pw $neo4j_password -co $colors\\n\",\n \"CALL apoc.meta.graph()\"\n ],\n \"metadata\": {\n \"id\": \"BfFOTNkncMqp\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"# Problem Definition\\n\"\n ],\n \"metadata\": {\n \"id\": \"d-x29ClTPBnv\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## What is Fraud?\\n\",\n \"Fraud occurs when an individual or group of individuals, or a business entity intentionally deceives another individual or business entity with misrepresentation of identity, products, services, or financial transactions and/or false promises with no intention of fulfilling them.\"\n ],\n \"metadata\": {\n \"id\": \"79q5QJfcPMa6\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Fraud Categories\\n\"\n ],\n \"metadata\": {\n \"id\": \"naUmXhC-PQGR\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### First-party Fraud\\n\",\n \"An individual, or group of individuals, misrepresent their identity or give false information when applying for a product or services to receive more favourable rates or when have no intention of repayment.\"\n ],\n \"metadata\": {\n \"id\": \"edTfWFSAPUKF\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Second-party Fraud\\n\",\n \"An individual knowingly gives their identity or personal information to another individual to commit fraud or someone is perpetrating fraud in his behalf.\"\n ],\n \"metadata\": {\n \"id\": \"Zr9sGs_9PYmH\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Third-party Fraud\\n\",\n \"An individual, or a group of individuals, create or use another person’s identity, or personal details, to open or takeover an account.\"\n ],\n \"metadata\": {\n \"id\": \"o45K16ryPcBu\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## The dataset\\n\",\n \"\\n\",\n \"We will use Paysim dataset for the hands-on exercises. Paysim is a synthetic dataset that mimics real world mobile money transfer network.\\n\",\n \"\\n\",\n \"For more information on the dataset, please visit this [blog page](https://www.sisu.io/posts/paysim/)\"\n ],\n \"metadata\": {\n \"id\": \"dFje1N1cPq_9\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"CALL apoc.meta.graph()\"\n ],\n \"metadata\": {\n \"id\": \"AAeicV33PDXa\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"There are five types of transactions in this database. List all transaction types and corresponding metrics by iterating over all the transactions.\"\n ],\n \"metadata\": {\n \"id\": \"Ux5tg_OzUvgT\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark\"\n ],\n \"metadata\": {\n \"id\": \"viWCvG1MU632\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"transaction_df = (spark.read\\n\",\n \" .format('org.neo4j.spark.DataSource')\\n\",\n \" .option('labels', ':Transaction')\\n\",\n \" .load())\\n\",\n \"\\n\",\n \"transaction_df_count = transaction_df.count()\\n\",\n \"\\n\",\n \"transaction_df = (transaction_df.groupBy('')\\n\",\n \" .count()\\n\",\n \" .withColumnRenamed('', 'transaction'))\\n\",\n \"\\n\",\n \"transaction_df = (transaction_df\\n\",\n \" .withColumn('transaction', transaction_df['transaction'].getItem(0))\\n\",\n \" .withColumn('% transactions', transaction_df['count'] / transaction_df_count))\\n\",\n \"\\n\",\n \"transaction_df.show(truncate=False)\"\n ],\n \"metadata\": {\n \"id\": \"xsIlmR-EQLeb\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark Pandas\"\n ],\n \"metadata\": {\n \"id\": \"U2JUpe4NU6Jz\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"transaction_ps = ps.read_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", options={\\\"labels\\\": \\\"Transaction\\\"})\\n\",\n \"\\n\",\n \"transaction_ps_count = transaction_ps.count()[0] * 1.0\\n\",\n \"\\n\",\n \"transaction_ps = (transaction_ps.groupby([''])\\n\",\n \" .size()\\n\",\n \" .reset_index(name='% transactions'))\\n\",\n \"\\n\",\n \"transaction_ps = transaction_ps.rename(columns={'': 'label'})\\n\",\n \"\\n\",\n \"transaction_ps['% transactions'] = transaction_ps['% transactions'].astype(float).div(transaction_ps_count * 1.0)\\n\",\n \"\\n\",\n \"transaction_ps.label = [x[0] for x in transaction_ps.label.to_numpy()]\\n\",\n \"\\n\",\n \"transaction_ps\"\n ],\n \"metadata\": {\n \"id\": \"rfkeCU0PVFXx\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"##### Plot the data\\n\",\n \"You can also use Python libraries like [Ploty](https://plotly.com/python/) to plot results\"\n ],\n \"metadata\": {\n \"id\": \"t2RufDichKkQ\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"import plotly.express as px\\n\",\n \"\\n\",\n \"# we use to_pandas() in order to transform the PySpark Pandas to a real Pandas Dataframe\\n\",\n \"fig = px.pie(transaction_ps.to_pandas(), values='% transactions', names='label')\\n\",\n \"\\n\",\n \"fig.show()\"\n ],\n \"metadata\": {\n \"id\": \"Wn4HMYGVhV1t\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"# Exploit first-party Fraud\\n\",\n \"\\n\",\n \"Synthetic identity fraud and first party fraud can be identified by performing entity link analysis to detect identities linked to other identities via shared PII.\\n\",\n \"\\n\",\n \"There are three types of personally identifiable information (PII) in this dataset - SSN, Email and Phone Number\\n\",\n \"\\n\",\n \"Our hypothesis is that clients who share identifiers are suspicious and have a higher potential to commit fraud. However, all shared identifier links are not suspicious, for example, two people sharing an email address. Hence, we compute a fraud score based on shared PII relationships and label the top X percentile clients as fraudsters.\\n\",\n \"\\n\",\n \"We will first identify clients that share identifiers and create a new relationship between clients that share identifiers\"\n ],\n \"metadata\": {\n \"id\": \"VwPKEtu2QLlv\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Enrich the dataset\"\n ],\n \"metadata\": {\n \"id\": \"E162NudWkW2n\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"In order to perfrorm our investigation we want to enrich the base dataset by identifing clients that share PII.\"\n ],\n \"metadata\": {\n \"id\": \"Y3MfFaKqH7Lm\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH (c1:Client)-[:HAS_EMAIL|:HAS_PHONE|:HAS_SSN]->(n)<-[:HAS_EMAIL|:HAS_PHONE|:HAS_SSN]-(c2:Client)\\n\",\n \"WHERE id(c1) < id(c2)\\n\",\n \"RETURN c1.id, c2.id, count(*) AS freq\\n\",\n \"ORDER BY freq DESC;\"\n ],\n \"metadata\": {\n \"id\": \"zJfRBlNNP9A1\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Now we can reuse the same Cypher query for creating our Dataframe and then use the Neo4j Spark Connector to create a new `SHARED_IDENTIFIERS` relationship betwen two clients:\\n\",\n \"\\n\",\n \"**(:Client)-[:SHARED_IDENTIFIERS]->(:Client)**\\n\",\n \"\\n\"\n ],\n \"metadata\": {\n \"id\": \"4J6d8U8bkMW_\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"// let's check if there relationships are in there\\n\",\n \"MATCH (c:Client)-[r:SHARED_IDENTIFIERS]->(c2:Client)\\n\",\n \"RETURN *\\n\",\n \"LIMIT 10\"\n ],\n \"metadata\": {\n \"id\": \"sQ7Nf_IUQQ1J\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"As you can see there are no relationships in the database\"\n ],\n \"metadata\": {\n \"id\": \"6zUwaYxgQhbC\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Code in PySpark\"\n ],\n \"metadata\": {\n \"id\": \"7CAiMFJ3LPmP\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"shared_identifiers_df = (spark.read.format(\\\"org.neo4j.spark.DataSource\\\")\\n\",\n \" .option(\\\"query\\\", \\\"\\\"\\\"\\n\",\n \" MATCH (c1:Client)-[:HAS_EMAIL|:HAS_PHONE|:HAS_SSN]->(n)<-[:HAS_EMAIL|:HAS_PHONE|:HAS_SSN]-(c2:Client)\\n\",\n \" WHERE id(c1) < id(c2)\\n\",\n \" RETURN c1.id AS source, c2.id AS target, count(*) AS freq\\n\",\n \" \\\"\\\"\\\")\\n\",\n \" .load())\\n\",\n \"\\n\",\n \"(shared_identifiers_df.write\\n\",\n \" .format(\\\"org.neo4j.spark.DataSource\\\")\\n\",\n \" .mode(\\\"Overwrite\\\")\\n\",\n \" .option(\\\"relationship\\\", \\\"SHARED_IDENTIFIERS\\\")\\n\",\n \" .option(\\\"relationship.save.strategy\\\", \\\"keys\\\")\\n\",\n \" .option(\\\"relationship.source.labels\\\", \\\":Client\\\")\\n\",\n \" .option(\\\"relationship.source.save.mode\\\", \\\"Overwrite\\\")\\n\",\n \" .option(\\\"relationship.source.node.keys\\\", \\\"source:id\\\")\\n\",\n \" .option(\\\"relationship.target.labels\\\", \\\":Client\\\")\\n\",\n \" .option(\\\"relationship.target.node.keys\\\", \\\"target:id\\\")\\n\",\n \" .option(\\\"relationship.target.save.mode\\\", \\\"Overwrite\\\")\\n\",\n \" .option(\\\"relationship.properties\\\", \\\"freq:count\\\")\\n\",\n \" .save())\"\n ],\n \"metadata\": {\n \"id\": \"36irb2nuj5Hi\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Code in PySpark Pandas\"\n ],\n \"metadata\": {\n \"id\": \"h07CRHnlLU-G\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"shared_identifiers_ps = ps.read_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", options={\\\"query\\\": \\\"\\\"\\\"\\n\",\n \" MATCH (c1:Client)-[:HAS_EMAIL|:HAS_PHONE|:HAS_SSN]->(n)<-[:HAS_EMAIL|:HAS_PHONE|:HAS_SSN]-(c2:Client)\\n\",\n \" WHERE id(c1) < id(c2)\\n\",\n \" RETURN c1.id AS source, c2.id AS target, count(*) AS freq\\n\",\n \"\\\"\\\"\\\"})\\n\",\n \"\\n\",\n \"shared_identifiers_ps.spark.to_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", mode=\\\"Overwrite\\\", options={\\n\",\n \" \\\"relationship\\\": \\\"SHARED_IDENTIFIERS\\\",\\n\",\n \" \\\"relationship.save.strategy\\\": \\\"keys\\\",\\n\",\n \" \\\"relationship.source.labels\\\": \\\":Client\\\",\\n\",\n \" \\\"relationship.source.save.mode\\\": \\\"Overwrite\\\",\\n\",\n \" \\\"relationship.source.node.keys\\\": \\\"source:id\\\",\\n\",\n \" \\\"relationship.target.labels\\\": \\\":Client\\\",\\n\",\n \" \\\"relationship.target.node.keys\\\": \\\"target:id\\\",\\n\",\n \" \\\"relationship.target.save.mode\\\": \\\"Overwrite\\\",\\n\",\n \" \\\"relationship.properties\\\": \\\"freq:count\\\"\\n\",\n \"})\"\n ],\n \"metadata\": {\n \"id\": \"vZ6So9x_MBY_\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"// let's check (again) if there relationships are in there\\n\",\n \"MATCH (c:Client)-[r:SHARED_IDENTIFIERS]->(c2:Client)\\n\",\n \"RETURN *\\n\",\n \"LIMIT 10\"\n ],\n \"metadata\": {\n \"id\": \"A6iUPMYMQAzF\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Build Fraud detection workflow in Neo4j GDS\\n\",\n \"\\n\",\n \"We will construct a workflow with graph algorithms to detect fraud rings, score clients based on the number of common connections and rank them to select the top few suspicious clients and label them as fraudsters.\\n\",\n \"\\n\",\n \"1. Identify clusters of clients sharing PII using a community detection algorithm (Weakly Connected Components)\\n\",\n \"2. Find similar clients within the clusters using pairwise similarity algorithms (Node Similarity)\\n\",\n \"3. Calculate and assign fraud score to clients using centrality algorithms (Degree Centrality)\\n\",\n \"4. Use computed fraud scores to label clients as potential fraudsters\"\n ],\n \"metadata\": {\n \"id\": \"Fy8roSDHQw3h\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Identify groups of clients sharing PII (Fraud rings)\\n\",\n \"\\n\",\n \"Run Weakly connected components to find clusters of clients sharing PII.\\n\",\n \"\\n\",\n \"Weakly Connected Components is used to find groups of connected nodes, where all nodes in the same set form a connected component. WCC is often used early in an analysis understand the structure of a graph. More informaton here: [WCC documentation](https://neo4j.com/docs/graph-data-science/current/algorithms/wcc/)\"\n ],\n \"metadata\": {\n \"id\": \"bysMYQ23WFVl\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Create a graph projection\\n\",\n \"\\n\",\n \"A central concept in the GDS library is the management of in-memory graphs. Graph algorithms run on a graph data model which is a projection of the Neo4j property graph data model. For more information, please click here: [Graph Management](https://neo4j.com/docs/graph-data-science/current/management-ops/)\\n\",\n \"\\n\",\n \"A projected graph can be stored in the catalog under a user-defined name. Using that name, the graph can be referred to by any algorithm in the library.\"\n ],\n \"metadata\": {\n \"id\": \"39g6Fq1dTgLt\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Consider that the original Cypher query is the following:\\n\",\n \"```cypher\\n\",\n \"CALL gds.graph.project('wcc',\\n\",\n \" {\\n\",\n \" Client: {\\n\",\n \" label: 'Client'\\n\",\n \" }\\n\",\n \" },\\n\",\n \" {\\n\",\n \" SHARED_IDENTIFIERS:{\\n\",\n \" type: 'SHARED_IDENTIFIERS',\\n\",\n \" orientation: 'UNDIRECTED',\\n\",\n \" properties: {\\n\",\n \" count: {\\n\",\n \" property: 'count'\\n\",\n \" }\\n\",\n \" }\\n\",\n \" }\\n\",\n \" }\\n\",\n \") YIELD graphName,nodeCount,relationshipCount,projectMillis;\\n\",\n \"```\\n\",\n \"\\n\",\n \"which will be translate into:\"\n ],\n \"metadata\": {\n \"id\": \"fXQwdpJfVGIq\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark\"\n ],\n \"metadata\": {\n \"id\": \"-fkvDjHcUV5Z\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"wcc_graph_proj_df = (spark.read.format(\\\"org.neo4j.spark.DataSource\\\")\\n\",\n \" .option(\\\"gds\\\", \\\"gds.graph.project\\\")\\n\",\n \" .option(\\\"gds.graphName\\\", \\\"wcc\\\")\\n\",\n \" .option(\\\"gds.nodeProjection.Client.label\\\", \\\"Client\\\")\\n\",\n \" .option(\\\"gds.relationshipProjection.SHARED_IDENTIFIERS.type\\\", \\\"SHARED_IDENTIFIERS\\\")\\n\",\n \" .option(\\\"gds.relationshipProjection.SHARED_IDENTIFIERS.orientation\\\", \\\"UNDIRECTED\\\")\\n\",\n \" .option(\\\"gds.relationshipProjection.SHARED_IDENTIFIERS.properties.count.property\\\", \\\"count\\\")\\n\",\n \" .load())\\n\",\n \"\\n\",\n \"wcc_graph_proj_df.show(truncate=False)\"\n ],\n \"metadata\": {\n \"id\": \"4dBHWHh8R7US\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark Pandas\"\n ],\n \"metadata\": {\n \"id\": \"cauhX4FRVYMy\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"wcc_graph_proj_ps = ps.read_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", options={\\n\",\n \" \\\"gds\\\": \\\"gds.graph.project\\\",\\n\",\n \" \\\"gds.graphName\\\": \\\"wcc\\\",\\n\",\n \" \\\"gds.nodeProjection.Client.label\\\": \\\"Client\\\",\\n\",\n \" \\\"gds.relationshipProjection.SHARED_IDENTIFIERS.type\\\": \\\"SHARED_IDENTIFIERS\\\",\\n\",\n \" \\\"gds.relationshipProjection.SHARED_IDENTIFIERS.orientation\\\": \\\"UNDIRECTED\\\",\\n\",\n \" \\\"gds.relationshipProjection.SHARED_IDENTIFIERS.properties.count.property\\\": \\\"count\\\"\\n\",\n \"})\\n\",\n \"\\n\",\n \"wcc_graph_proj_ps\"\n ],\n \"metadata\": {\n \"id\": \"ZndBGbXPVeqU\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Run the WCC algorithm\\n\",\n \"\\n\",\n \"The original Cypher query is:\\n\",\n \"\\n\",\n \"```cypher\\n\",\n \"CALL gds.wcc.stream('wcc',\\n\",\n \" {\\n\",\n \" nodeLabels: ['Client'],\\n\",\n \" relationshipTypes: ['SHARED_IDENTIFIERS'],\\n\",\n \" consecutiveIds: true\\n\",\n \" }\\n\",\n \")\\n\",\n \"YIELD nodeId, componentId\\n\",\n \"RETURN gds.util.asNode(nodeId).id AS clientId, componentId\\n\",\n \"ORDER BY componentId\\n\",\n \"LIMIT 20\\n\",\n \"```\\n\",\n \"\\n\",\n \"which is transate into:\"\n ],\n \"metadata\": {\n \"id\": \"P4oIKsUNn-ZH\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark\"\n ],\n \"metadata\": {\n \"id\": \"Ygw7T3lSWbsQ\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# get the clients\\n\",\n \"clients_df = (spark.read.format(\\\"org.neo4j.spark.DataSource\\\")\\n\",\n \" .option(\\\"labels\\\", \\\"Client\\\")\\n\",\n \" .load())\\n\",\n \"\\n\",\n \"# invoke the gds wcc stream procedure\\n\",\n \"wcc_df = (spark.read.format(\\\"org.neo4j.spark.DataSource\\\")\\n\",\n \" .option(\\\"gds\\\", \\\"gds.wcc.stream\\\")\\n\",\n \" .option(\\\"gds.graphName\\\", \\\"wcc\\\")\\n\",\n \" .option(\\\"gds.nodeLabels\\\", \\\"['Client']\\\")\\n\",\n \" .option(\\\"gds.relationshipTypes\\\", \\\"['SHARED_IDENTIFIERS']\\\")\\n\",\n \" .option(\\\"gds.consecutiveIds\\\", \\\"true\\\")\\n\",\n \" .load())\\n\",\n \"\\n\",\n \"# join the two dataframes and show id, componentId\\n\",\n \"client_component_df = (clients_df.join(wcc_df, clients_df[\\\"\\\"] == wcc_df[\\\"nodeId\\\"], \\\"inner\\\")\\n\",\n \" .select(\\\"id\\\", \\\"componentId\\\"))\\n\",\n \"\\n\",\n \"client_component_df.show(truncate=False)\"\n ],\n \"metadata\": {\n \"id\": \"6RtkJV9GWHnu\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark Pandas\"\n ],\n \"metadata\": {\n \"id\": \"xQI9gQNQYMWe\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# get the clients\\n\",\n \"clients_ps = ps.read_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", options={\\\"labels\\\": \\\"Client\\\"})\\n\",\n \"\\n\",\n \"# invoke the gds wcc stream procedure\\n\",\n \"wcc_ps = ps.read_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", options={\\n\",\n \" \\\"gds\\\": \\\"gds.wcc.stream\\\",\\n\",\n \" \\\"gds.graphName\\\": \\\"wcc\\\",\\n\",\n \" \\\"gds.nodeLabels\\\": \\\"['Client']\\\",\\n\",\n \" \\\"gds.relationshipTypes\\\": \\\"['SHARED_IDENTIFIERS']\\\",\\n\",\n \" \\\"gds.consecutiveIds\\\": \\\"true\\\"\\n\",\n \"})\\n\",\n \"\\n\",\n \"# join the two pandas df and show id, componentId\\n\",\n \"client_component_ps = clients_ps.join(wcc_ps.set_index(\\\"nodeId\\\"), on=\\\"\\\")[[\\\"id\\\", \\\"componentId\\\"]]\\n\",\n \"\\n\",\n \"# we show only the first 20\\n\",\n \"client_component_ps[:20]\"\n ],\n \"metadata\": {\n \"id\": \"AvlUnpIQYQsZ\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Write results to the database.\\n\",\n \"Now that we identified clusters of clients sharing PII, we want to store these results back into the database by enriching the `Client` node.\\n\",\n \"We'll add the component id of the cluster as `firstPartyFraudGroup` property\"\n ],\n \"metadata\": {\n \"id\": \"eNgmAuheZqfA\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark\"\n ],\n \"metadata\": {\n \"id\": \"EKNcuklDaRKY\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"(client_component_df\\n\",\n \" .withColumnRenamed(\\\"componentId\\\", \\\"firstPartyFraudGroup\\\")\\n\",\n \" .write\\n\",\n \" .format(\\\"org.neo4j.spark.DataSource\\\")\\n\",\n \" .mode(\\\"Overwrite\\\")\\n\",\n \" .option(\\\"labels\\\", \\\"Client\\\")\\n\",\n \" .option(\\\"node.keys\\\", \\\"id\\\")\\n\",\n \" .save())\"\n ],\n \"metadata\": {\n \"id\": \"yQb0H-p7ZrRP\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark Pandas\"\n ],\n \"metadata\": {\n \"id\": \"-HK28CGoa5_p\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"(client_component_ps\\n\",\n \" .rename(columns={\\\"componentId\\\": \\\"firstPartyFraudGroup\\\"})\\n\",\n \" .spark\\n\",\n \" .to_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", mode=\\\"Overwrite\\\", options={\\n\",\n \" \\\"labels\\\": \\\"Client\\\",\\n\",\n \" \\\"node.keys\\\": \\\"id\\\"\\n\",\n \" }))\"\n ],\n \"metadata\": {\n \"id\": \"zJDS-0bta8_s\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"// Visualize clusters with greater than 9 client nodes.\\n\",\n \"MATCH (c:Client)\\n\",\n \"WITH c.firstPartyFraudGroup AS fpGroupID, collect(c.id) AS fGroup\\n\",\n \"WITH *, size(fGroup) AS groupSize WHERE groupSize >= 9\\n\",\n \"WITH * LIMIT 1\\n\",\n \"MATCH p=(c:Client)-[:HAS_SSN|HAS_EMAIL|HAS_PHONE]->()\\n\",\n \"WHERE c.firstPartyFraudGroup = fpGroupID\\n\",\n \"RETURN p\"\n ],\n \"metadata\": {\n \"id\": \"oOsrNUZocx21\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Compute pairwise similarity scores\\n\",\n \"\\n\",\n \"We use node similarity algorithm to find similar nodes based on the relationships to other nodes. Node similarity uses Jaccard metric ([Node Similarity](https://neo4j.com/docs/graph-data-science/current/algorithms/node-similarity/#algorithms-node-similarity))\\n\",\n \"\\n\",\n \"Node similarity algorithms work on bipartite graphs (two types of nodes and relationships between them). Here we project client nodes (one type) and three identifiers nodes (that are considered as second type) into memory.\"\n ],\n \"metadata\": {\n \"id\": \"5CCwYp1FfoMU\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Project the graph\\n\",\n \"\\n\",\n \"The original Cypher query is\\n\",\n \"\\n\",\n \"```cypher\\n\",\n \"MATCH(c:Client) WHERE c.firstPartyFraudGroup is not NULL\\n\",\n \"WITH collect(c) as clients\\n\",\n \"MATCH(n) WHERE n:Email OR n:Phone OR n:SSN\\n\",\n \"WITH clients, collect(n) as identifiers\\n\",\n \"WITH clients + identifiers as nodes\\n\",\n \"\\n\",\n \"MATCH(c:Client) -[:HAS_EMAIL|:HAS_PHONE|:HAS_SSN]->(id)\\n\",\n \"WHERE c.firstPartyFraudGroup is not NULL\\n\",\n \"WITH nodes, collect({source: c, target: id}) as relationships\\n\",\n \"\\n\",\n \"CALL gds.graph.project.cypher('similarity',\\n\",\n \" \\\"UNWIND $nodes as n RETURN id(n) AS id,labels(n) AS labels\\\",\\n\",\n \" \\\"UNWIND $relationships as r RETURN id(r['source']) AS source, id(r['target']) AS target, 'HAS_IDENTIFIER' as type\\\",\\n\",\n \" { parameters: {nodes: nodes, relationships: relationships}}\\n\",\n \")\\n\",\n \"YIELD graphName, nodeCount, relationshipCount, projectMillis\\n\",\n \"RETURN graphName, nodeCount, relationshipCount, projectMillis\\n\",\n \"```\\n\",\n \"\\n\",\n \"Which is translated into\"\n ],\n \"metadata\": {\n \"id\": \"aLGQxFtpnQHa\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark\"\n ],\n \"metadata\": {\n \"id\": \"aPSY4htNgLVG\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"similarity_graph_proj_df = (spark.read.format(\\\"org.neo4j.spark.DataSource\\\")\\n\",\n \" .option(\\\"gds\\\", \\\"gds.graph.project.cypher\\\")\\n\",\n \" .option(\\\"gds.graphName\\\", \\\"similarity\\\")\\n\",\n \" .option(\\\"gds.nodeQuery\\\", \\\"\\\"\\\"\\n\",\n \" MATCH (n)\\n\",\n \" WHERE (n:Client AND n.firstPartyFraudGroup is not NULL) OR n:Email OR n:Phone OR n:SSN\\n\",\n \" RETURN id(n) AS id, labels(n) AS labels\\n\",\n \" \\\"\\\"\\\")\\n\",\n \" .option(\\\"gds.relationshipQuery\\\", \\\"\\\"\\\"\\n\",\n \" MATCH (s:Client)-[:HAS_EMAIL|:HAS_PHONE|:HAS_SSN]->(t)\\n\",\n \" WHERE s.firstPartyFraudGroup is not NULL\\n\",\n \" RETURN id(s) AS source, id(t) AS target, 'HAS_IDENTIFIER' as type\\n\",\n \" \\\"\\\"\\\")\\n\",\n \" .load())\\n\",\n \"\\n\",\n \"similarity_graph_proj_df.show(truncate=False)\"\n ],\n \"metadata\": {\n \"id\": \"eNvVkJTRfuqM\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark Pandas\"\n ],\n \"metadata\": {\n \"id\": \"PcJCKDDhmbtr\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"similarity_graph_proj_ps = ps.read_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", options={\\n\",\n \" \\\"gds\\\": \\\"gds.graph.project.cypher\\\",\\n\",\n \" \\\"gds.graphName\\\": \\\"similarity\\\",\\n\",\n \" \\\"gds.nodeQuery\\\": \\\"\\\"\\\"\\n\",\n \" MATCH (n)\\n\",\n \" WHERE (n:Client AND n.firstPartyFraudGroup is not NULL) OR n:Email OR n:Phone OR n:SSN\\n\",\n \" RETURN id(n) AS id, labels(n) AS labels\\n\",\n \" \\\"\\\"\\\",\\n\",\n \" \\\"gds.relationshipQuery\\\": \\\"\\\"\\\"\\n\",\n \" MATCH (s:Client)-[:HAS_EMAIL|:HAS_PHONE|:HAS_SSN]->(t)\\n\",\n \" WHERE s.firstPartyFraudGroup is not NULL\\n\",\n \" RETURN id(s) AS source, id(t) AS target, 'HAS_IDENTIFIER' as type\\n\",\n \" \\\"\\\"\\\"\\n\",\n \"})\\n\",\n \"\\n\",\n \"similarity_graph_proj_ps\"\n ],\n \"metadata\": {\n \"id\": \"odriamHwmiEQ\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Compute the node similarity\\n\",\n \"\\n\",\n \"We can mutate in-memory graph by writing outputs from the algorithm as node or relationship properties.\\n\",\n \"\\n\",\n \"In this particular case all the procedures with `mutate` and `write` suffix are not supported from the Neo4j Spark Connector, in this case we'll write a Cypher query:\\n\"\n ],\n \"metadata\": {\n \"id\": \"TfnVREvvmzG4\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"CALL gds.nodeSimilarity.mutate('similarity',\\n\",\n \" {\\n\",\n \" topK:15,\\n\",\n \" mutateProperty: 'jaccardScore',\\n\",\n \" mutateRelationshipType:'SIMILAR_TO'\\n\",\n \" }\\n\",\n \");\"\n ],\n \"metadata\": {\n \"id\": \"E0_WoCLlnIr6\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Mutate mode is very fast compared to write mode and it helps in optimizing algorithm execution times, then we write back the property from in-memory graph to the database and use it for further analysis:\"\n ],\n \"metadata\": {\n \"id\": \"JnC3C7urPyN_\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"CALL gds.graph.writeRelationship('similarity', 'SIMILAR_TO', 'jaccardScore');\"\n ],\n \"metadata\": {\n \"id\": \"xajV8enLPvIN\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Exercise: Calculate First-party Fraud Score\\n\",\n \"\\n\",\n \"We compute first party fraud score using weighted degree centrality algorithm.\\n\",\n \"\\n\",\n \"In this step, we compute and assign fraud score (`firstPartyFraudScore`) to clients in the clusters identified in previous steps based on `SIMILAR_TO` relationships weighted by `jaccardScore`\\n\",\n \"\\n\",\n \"Weighted degree centrality algorithm add up similarity scores (`jaccardScore`) on the incoming `SIMILAR_TO` relationships for a given node in a cluster and assign the sum as the corresponding `firstPartyFraudScore`. This score represents clients who are similar to many others in the cluster in terms of sharing identifiers. Higher `firstPartyFraudScore` represents greater potential for committing fraud.\"\n ],\n \"metadata\": {\n \"id\": \"qpXXoWeIQk9U\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Code in PySpark\"\n ],\n \"metadata\": {\n \"id\": \"fAp_acV-RBOu\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# invoke the gds.degree.stream procedure\"\n ],\n \"metadata\": {\n \"id\": \"ZPAHXvT6Qouy\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"similarity_df = (spark.read.format(\\\"org.neo4j.spark.DataSource\\\")\\n\",\n \" .option(\\\"gds\\\", \\\"gds.degree.stream\\\")\\n\",\n \" .option(\\\"gds.graphName\\\", \\\"similarity\\\")\\n\",\n \" .option(\\\"gds.nodeLabels\\\", \\\"['Client']\\\")\\n\",\n \" .option(\\\"gds.relationshipTypes\\\", \\\"['SIMILAR_TO']\\\")\\n\",\n \" .option(\\\"gds.relationshipWeightProperty\\\", \\\"jaccardScore\\\")\\n\",\n \" .load())\\n\",\n \"\\n\",\n \"# join the two dataframes and show id, score\\n\",\n \"client_similarity_df = (clients_df.join(similarity_df, clients_df[\\\"\\\"] == similarity_df[\\\"nodeId\\\"], \\\"inner\\\")\\n\",\n \" .select(\\\"id\\\", \\\"score\\\")\\n\",\n \" .withColumnRenamed(\\\"score\\\", \\\"firstPartyFraudScore\\\"))\\n\",\n \"\\n\",\n \"# write the results back to the database\\n\",\n \"(client_similarity_df.write.format('org.neo4j.spark.DataSource')\\n\",\n \" .mode(\\\"Overwrite\\\")\\n\",\n \" .option(\\\"labels\\\", \\\"Client\\\")\\n\",\n \" .option(\\\"node.keys\\\", \\\"id\\\")\\n\",\n \" .save())\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\"\n ],\n \"metadata\": {\n \"id\": \"1qKbMW3FSwLE\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Code in PySpark Pandas\\n\",\n \"\\n\"\n ],\n \"metadata\": {\n \"id\": \"na2h6VLov-ZA\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# invoke the gds.degree.stream procedure\"\n ],\n \"metadata\": {\n \"id\": \"7ObUlkdmwDzi\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"similarity_ps = ps.read_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", options={\\n\",\n \" \\\"gds\\\": \\\"gds.degree.stream\\\",\\n\",\n \" \\\"gds.graphName\\\": \\\"similarity\\\",\\n\",\n \" \\\"gds.nodeLabels\\\": \\\"['Client']\\\",\\n\",\n \" \\\"gds.relationshipTypes\\\": \\\"['SIMILAR_TO']\\\",\\n\",\n \" \\\"gds.relationshipWeightProperty\\\": \\\"jaccardScore\\\"\\n\",\n \"})\\n\",\n \"\\n\",\n \"# join the two pandas df and show id, score\\n\",\n \"client_similarity_ps = (clients_ps.join(similarity_ps.set_index(\\\"nodeId\\\"), on=\\\"\\\")[[\\\"id\\\", \\\"score\\\"]]\\n\",\n \" .rename(columns={\\\"score\\\": \\\"firstPartyFraudScore\\\"}))\\n\",\n \"\\n\",\n \"# write the results back to the database\\n\",\n \"client_similarity_ps.spark.to_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", mode=\\\"Overwrite\\\", options={\\n\",\n \" \\\"labels\\\": \\\"Client\\\",\\n\",\n \" \\\"node.keys\\\": \\\"id\\\"\\n\",\n \"})\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\"\n ],\n \"metadata\": {\n \"id\": \"m8Cy4NtsS3nQ\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Verifiy the result\\n\",\n \"\\n\",\n \"We expect that:\\n\",\n \"- `similarity_df`/`similarity_ps`\\n\",\n \" - has two columns:\\n\",\n \" - `nodeId` of long type\\n\",\n \" - `score` of double type\\n\",\n \" - a count of **9134** rows\\n\",\n \"- `client_similarity_df`/`client_similarity_ps`\\n\",\n \" - has two columns:\\n\",\n \" - `id` of long type\\n\",\n \" - `score` of double type\\n\",\n \" - a count of 2433 rows\"\n ],\n \"metadata\": {\n \"id\": \"I_TKq9TmU3cS\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Test PySpark Dataframe\"\n ],\n \"metadata\": {\n \"id\": \"Dow9PWAtxBPu\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"assert StructType([StructField(\\\"nodeId\\\", LongType()), StructField(\\\"score\\\", DoubleType())]) == similarity_df.schema\\n\",\n \"assert 9134 == similarity_df.count()\\n\",\n \"\\n\",\n \"assert StructType([StructField(\\\"id\\\", StringType()), StructField(\\\"firstPartyFraudScore\\\", DoubleType())]) == client_similarity_df.schema\\n\",\n \"assert 2433 == client_similarity_df.count()\\n\",\n \"print(\\\"All assertion are successfuly satisfied.\\\")\"\n ],\n \"metadata\": {\n \"id\": \"5E6fUHwZU7PN\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Test PySpark Pandas\"\n ],\n \"metadata\": {\n \"id\": \"feuIfTCqxGKt\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"assert StructType([StructField(\\\"nodeId\\\", LongType()), StructField(\\\"score\\\", DoubleType())]) == similarity_ps.to_spark().schema\\n\",\n \"assert 9134 == similarity_ps.count()[0]\\n\",\n \"\\n\",\n \"assert StructType([StructField(\\\"id\\\", StringType()), StructField(\\\"firstPartyFraudScore\\\", DoubleType())]) == client_similarity_ps.to_spark().schema\\n\",\n \"assert 2433 == client_similarity_ps.count()[0]\\n\",\n \"print(\\\"All assertion are successfuly satisfied.\\\")\"\n ],\n \"metadata\": {\n \"id\": \"OHel2EYoxFpC\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"We find clients with first-party fraud score greater than some threshold (X) and label those top X percentile clients as fraudsters. In this example, using 95th percentile as a threshold, we set a property FirstPartyFraudster on the Client node.\"\n ],\n \"metadata\": {\n \"id\": \"YBhfX5DYXsom\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH (c:Client)\\n\",\n \"WHERE c.firstPartyFraudScore IS NOT NULL\\n\",\n \"WITH percentileCont(c.firstPartyFraudScore, 0.95) AS firstPartyFraudThreshold\\n\",\n \"MATCH (c:Client)\\n\",\n \"WHERE c.firstPartyFraudScore > firstPartyFraudThreshold\\n\",\n \"SET c:FirstPartyFraudster\"\n ],\n \"metadata\": {\n \"id\": \"bSb4rZ-aXvYV\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"# Second-party Fraud / Money Mules\\n\",\n \"\\n\",\n \"The first step is to find out clients who weren't identified as first party fraudsters but they transact with first party fraudsters.\"\n ],\n \"metadata\": {\n \"id\": \"WyKyntZ_YC1g\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH p=(:Client:FirstPartyFraudster)-[]-(:Transaction)-[]-(c:Client)\\n\",\n \"WHERE NOT c:FirstPartyFraudster\\n\",\n \"RETURN p\\n\",\n \"LIMIT 50\"\n ],\n \"metadata\": {\n \"id\": \"C94Yg_psYTRW\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Also, lets find out what types of transactions do these Clients perform with first party fraudsters\"\n ],\n \"metadata\": {\n \"id\": \"Tm9WYZnwsQif\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH (:Client:FirstPartyFraudster)-[]-(txn:Transaction)-[]-(c:Client)\\n\",\n \"WHERE NOT c:FirstPartyFraudster\\n\",\n \"UNWIND labels(txn) AS transactionType\\n\",\n \"RETURN transactionType, count(*) AS freq\"\n ],\n \"metadata\": {\n \"id\": \"x2P9Y7IzY2Vl\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Create new relationships\\n\",\n \"\\n\",\n \"Let’s go ahead and create `TRANSFER_TO` relationships between clients with `firstPartyFraudster` tags and other clients. Also add the total amount from all such transactions as a property on `TRANSFER_TO` relationships.\\n\",\n \"\\n\",\n \"Since the total amount transferred from a fraudster to a client and the total amount transferred in the reverse direction are not the same, we have to create relationships in two separate queries.\\n\",\n \"\\n\",\n \"* `TRANSFER_TO` relationship from a fraudster to a client (look at the directions in queries)\\n\",\n \"* Add `SecondPartyFraudSuspect` tag to these clients\"\n ],\n \"metadata\": {\n \"id\": \"f2HTap1vuLBW\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH (c1:FirstPartyFraudster)-[]->(t:Transaction)-[]->(c2:Client)\\n\",\n \"WHERE NOT c2:FirstPartyFraudster\\n\",\n \"WITH c1, c2, sum(t.amount) AS totalAmount\\n\",\n \"SET c2:SecondPartyFraudSuspect\\n\",\n \"CREATE (c1)-[:TRANSFER_TO {amount:totalAmount}]->(c2)\"\n ],\n \"metadata\": {\n \"id\": \"lfGkbJk2ueau\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"* `TRANSFER_TO` relationship from a client to a fraudster.\"\n ],\n \"metadata\": {\n \"id\": \"6yO2wJP6uksA\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH (c1:FirstPartyFraudster)<-[]-(t:Transaction)<-[]-(c2:Client)\\n\",\n \"WHERE NOT c2:FirstPartyFraudster\\n\",\n \"WITH c1, c2, sum(t.amount) AS totalAmount\\n\",\n \"SET c2:SecondPartyFraudSuspect\\n\",\n \"CREATE (c1)<-[:TRANSFER_TO {amount:totalAmount}]-(c2);\"\n ],\n \"metadata\": {\n \"id\": \"WUeeP4MhujzM\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"Visualize newly created `TRANSFER_TO` relationships\"\n ],\n \"metadata\": {\n \"id\": \"68rcS85lutUx\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"%%cypher\\n\",\n \"MATCH p=(:Client:FirstPartyFraudster)-[:TRANSFER_TO]-(c:Client)\\n\",\n \"WHERE NOT c:FirstPartyFraudster\\n\",\n \"RETURN p\\n\",\n \"LIMIT 50\"\n ],\n \"metadata\": {\n \"id\": \"LC5kSXOGuu1w\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Goal\\n\",\n \"\\n\",\n \"Our objective is to find out clients who may have supported the first party fraudsters and were not identified as potential first party fraudsters.\\n\",\n \"\\n\",\n \"Our hypothesis is that clients who perform transactions of type `Transfer` where they either send or receive money from first party fraudsters are flagged as suspects for second party fraud.\\n\",\n \"\\n\",\n \"To identify such clients, make use of `TRANSFER_TO` relationships and use this recipe:\\n\",\n \"\\n\",\n \"* Use **WCC** (community detection) to identify networks of clients who are connected to first party fraudsters\\n\",\n \"* Use **PageRank** (centrality) to score clients based on their influence in terms of the amount of money transferred to/from fraudsters\\n\",\n \"* Assign risk score (`secondPartyFraudScore`) to these clients\"\n ],\n \"metadata\": {\n \"id\": \"6swyguhivCEL\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Exercise: Project the graph\\n\",\n \"\\n\",\n \"Let’s use native projection and create an in-memory graph with Client nodes and TRANSFER_TO relationships.\\n\",\n \"\\n\",\n \"We want to project:\\n\",\n \"* `Client` for `nodeProjection`\\n\",\n \"* `TRANSFER_TO` for `relationshipProjection`\\n\",\n \"* for the configuration we want to set `relationshipProperties` to `amount`\"\n ],\n \"metadata\": {\n \"id\": \"ILntBuwxvXCz\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Code in PySpark\"\n ],\n \"metadata\": {\n \"id\": \"zgKufo0Z2C8e\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"second_party_graph_proj_df = # insert your PySpark code here\"\n ],\n \"metadata\": {\n \"id\": \"JmhR2nhwvWRu\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"second_party_graph_proj_df = (spark.read.format(\\\"org.neo4j.spark.DataSource\\\")\\n\",\n \" .option(\\\"gds\\\", \\\"gds.graph.project\\\")\\n\",\n \" .option(\\\"gds.graphName\\\", \\\"SecondPartyFraudNetwork\\\")\\n\",\n \" .option(\\\"gds.nodeProjection\\\", \\\"Client\\\")\\n\",\n \" .option(\\\"gds.relationshipProjection\\\", \\\"TRANSFER_TO\\\")\\n\",\n \" .option(\\\"gds.configuration.relationshipProperties\\\", \\\"amount\\\")\\n\",\n \" .load())\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\"\n ],\n \"metadata\": {\n \"id\": \"Ab6GEhJ_Tsrx\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Code in PySpark Pandas\"\n ],\n \"metadata\": {\n \"id\": \"QHnI8W5J2Gcp\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"second_party_graph_proj_ps = # insert your PySpark Pandas code here\"\n ],\n \"metadata\": {\n \"id\": \"kD6vXaab3JdM\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"second_party_graph_proj_ps = ps.read_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", options={\\n\",\n \" \\\"gds\\\": \\\"gds.graph.project\\\",\\n\",\n \" \\\"gds.graphName\\\": \\\"SecondPartyFraudNetwork\\\",\\n\",\n \" \\\"gds.nodeProjection\\\": \\\"Client\\\",\\n\",\n \" \\\"gds.relationshipProjection\\\": \\\"TRANSFER_TO\\\",\\n\",\n \" \\\"gds.configuration.relationshipProperties\\\": \\\"amount\\\"\\n\",\n \"})\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\"\n ],\n \"metadata\": {\n \"id\": \"c0VjV2rBTy10\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Verify the projection\"\n ],\n \"metadata\": {\n \"id\": \"ZVul68Wi162b\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Test PySpark Dataframe\"\n ],\n \"metadata\": {\n \"id\": \"jlOuUGA833ny\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"second_party_graph_proj_df.cache()\\n\",\n \"\\n\",\n \"first_row = [\\n\",\n \" {\\n\",\n \" \\\"node\\\": list(row[\\\"nodeProjection\\\"].keys())[0],\\n\",\n \" \\\"rel\\\": list(row[\\\"relationshipProjection\\\"].keys())[0],\\n\",\n \" \\\"graphName\\\": row[\\\"graphName\\\"],\\n\",\n \" \\\"nodeCount\\\": row[\\\"nodeCount\\\"],\\n\",\n \" \\\"relCount\\\": row[\\\"relationshipCount\\\"]\\n\",\n \" } for row in second_party_graph_proj_df.collect()\\n\",\n \"][0]\\n\",\n \"\\n\",\n \"assert first_row == {\\\"node\\\": \\\"Client\\\", \\\"rel\\\": \\\"TRANSFER_TO\\\", \\\"graphName\\\": \\\"SecondPartyFraudNetwork\\\", \\\"nodeCount\\\": 2433, \\\"relCount\\\": 367}\\n\",\n \"print(\\\"All assertion are successfuly satisfied.\\\")\"\n ],\n \"metadata\": {\n \"id\": \"hkjx6Nu339wW\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Test PySpark Pandas\"\n ],\n \"metadata\": {\n \"id\": \"37mDMX8G4EPE\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"second_party_graph_proj_ps_df = second_party_graph_proj_ps.to_spark()\\n\",\n \"\\n\",\n \"second_party_graph_proj_ps_df.cache()\\n\",\n \"\\n\",\n \"first_row = [\\n\",\n \" {\\n\",\n \" \\\"node\\\": list(row[\\\"nodeProjection\\\"].keys())[0],\\n\",\n \" \\\"rel\\\": list(row[\\\"relationshipProjection\\\"].keys())[0],\\n\",\n \" \\\"graphName\\\": row[\\\"graphName\\\"],\\n\",\n \" \\\"nodeCount\\\": row[\\\"nodeCount\\\"],\\n\",\n \" \\\"relCount\\\": row[\\\"relationshipCount\\\"]\\n\",\n \" } for row in second_party_graph_proj_ps_df.collect()\\n\",\n \"][0]\\n\",\n \"\\n\",\n \"assert first_row == {\\\"node\\\": \\\"Client\\\", \\\"rel\\\": \\\"TRANSFER_TO\\\", \\\"graphName\\\": \\\"SecondPartyFraudNetwork\\\", \\\"nodeCount\\\": 2433, \\\"relCount\\\": 367}\\n\",\n \"print(\\\"All assertion are successfuly satisfied.\\\")\"\n ],\n \"metadata\": {\n \"id\": \"spF535cp4IcR\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"## Check clusters with more than one clients\\n\",\n \"We will see if there are any clusters with more than one clients in them and if there are, then we should add a tag `secondPartyFraudGroup` to find them later using local queries.\"\n ],\n \"metadata\": {\n \"id\": \"32FL1L324QxV\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Code in PySpark\"\n ],\n \"metadata\": {\n \"id\": \"UljCEN-s40mc\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# invoke gds.wcc.stream on\\n\",\n \"second_party_wcc_df = (spark.read.format(\\\"org.neo4j.spark.DataSource\\\")\\n\",\n \" .option(\\\"gds\\\", \\\"gds.wcc.stream\\\")\\n\",\n \" .option(\\\"gds.graphName\\\", \\\"SecondPartyFraudNetwork\\\")\\n\",\n \" .load())\\n\",\n \"\\n\",\n \"# join the two dataframes aggregate by componentId\\n\",\n \"# and filtering for clusters with a size greater then 1\\n\",\n \"second_party_client_component_df = (clients_df.join(second_party_wcc_df, clients_df[\\\"\\\"] == second_party_wcc_df[\\\"nodeId\\\"], \\\"inner\\\")\\n\",\n \" .groupBy(\\\"componentId\\\")\\n\",\n \" .agg(count(\\\"*\\\").alias(\\\"count\\\"), collect_list(clients_df[\\\"id\\\"]).alias(\\\"cluster\\\"))\\n\",\n \" .filter(size(\\\"cluster\\\") > 1))\\n\",\n \"\\n\",\n \"second_party_client_component_df = (second_party_client_component_df\\n\",\n \" .withColumn(\\\"id\\\", explode(col(\\\"cluster\\\")))\\n\",\n \" .select(\\\"id\\\", \\\"componentId\\\"))\\n\",\n \"\\n\",\n \"\\n\",\n \"second_party_client_component_df.show(truncate=False)\"\n ],\n \"metadata\": {\n \"id\": \"jKK3m8rH150-\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Code in PySpark Pandas\"\n ],\n \"metadata\": {\n \"id\": \"cL0e6QHDZZZi\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"second_party_wcc_ps = ps.read_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", options={\\n\",\n \" \\\"gds\\\": \\\"gds.wcc.stream\\\",\\n\",\n \" \\\"gds.graphName\\\": \\\"SecondPartyFraudNetwork\\\"\\n\",\n \"})\\n\",\n \"\\n\",\n \"second_party_client_component_ps = (clients_ps.join(second_party_wcc_ps.set_index(\\\"nodeId\\\"), on=\\\"\\\")\\n\",\n \" .groupby([\\\"componentId\\\"])\\n\",\n \" .id\\n\",\n \" .apply(list)\\n\",\n \" .reset_index()\\n\",\n \" .rename(columns={'id': 'cluster'})\\n\",\n \")\\n\",\n \"\\n\",\n \"second_party_client_component_ps = second_party_client_component_ps[second_party_client_component_ps[\\\"cluster\\\"].apply(lambda a: len(a)) > 1]\\n\",\n \"\\n\",\n \"second_party_client_component_ps = (second_party_client_component_ps\\n\",\n \" .explode(\\\"cluster\\\")\\n\",\n \" .rename(columns={'cluster': 'id'})\\n\",\n \" [[\\\"id\\\", \\\"componentId\\\"]])\\n\",\n \"\\n\",\n \"second_party_client_component_ps[:20]\"\n ],\n \"metadata\": {\n \"id\": \"49OxrIuTZdb-\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"### Exercise: Write the results back to database\\n\",\n \"\\n\",\n \"Write a Spark job that given the columns (`id`, `componentId`) retrieve the `Client` by the `id` and set for the node a property `secondPartyFraudGroup` with the value of `componentId`\"\n ],\n \"metadata\": {\n \"id\": \"9IsIbHD7b3cL\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark\"\n ],\n \"metadata\": {\n \"id\": \"1U2kzCyWdv8l\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# write your PySpark code here\"\n ],\n \"metadata\": {\n \"id\": \"PTid4oiqg_lu\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"(second_party_client_component_df\\n\",\n \" .withColumnRenamed(\\\"componentId\\\", \\\"secondPartyFraudGroup\\\")\\n\",\n \" .write\\n\",\n \" .format(\\\"org.neo4j.spark.DataSource\\\")\\n\",\n \" .mode(\\\"Overwrite\\\")\\n\",\n \" .option(\\\"labels\\\", \\\"Client\\\")\\n\",\n \" .option(\\\"node.keys\\\", \\\"id\\\")\\n\",\n \" .save())\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\"\n ],\n \"metadata\": {\n \"id\": \"BQB9o9JCYuOs\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Code in PySpark Pandas\"\n ],\n \"metadata\": {\n \"id\": \"QgUs-jzIdyWJ\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"# write your PySpark Pandas code here\"\n ],\n \"metadata\": {\n \"id\": \"Copd3BPHduvJ\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"
\\n\",\n \"\\n\",\n \"Show a possible solution\\n\",\n \"\\n\",\n \"\\n\",\n \"```python\\n\",\n \"(client_component_ps\\n\",\n \" .rename(columns={\\\"componentId\\\": \\\"secondPartyFraudGroup\\\"})\\n\",\n \" .spark\\n\",\n \" .to_spark_io(format=\\\"org.neo4j.spark.DataSource\\\", mode=\\\"Overwrite\\\", options={\\n\",\n \" \\\"labels\\\": \\\"Client\\\",\\n\",\n \" \\\"node.keys\\\": \\\"id\\\"\\n\",\n \" }))\\n\",\n \"```\\n\",\n \"\\n\",\n \"
\"\n ],\n \"metadata\": {\n \"id\": \"7uiMcrNHY3GS\"\n }\n },\n {\n \"cell_type\": \"markdown\",\n \"source\": [\n \"#### Verify the Spark job result\"\n ],\n \"metadata\": {\n \"id\": \"0mi9x4ragzyX\"\n }\n },\n {\n \"cell_type\": \"code\",\n \"source\": [\n \"secondPartyFraudGroup_check_count = %cypher -u $neo4j_url MATCH (c:Client) WHERE c.secondPartyFraudGroup IS NOT NULL RETURN count(c) AS count\\n\",\n \"assert 2433 == secondPartyFraudGroup_check_count['count'][0]\\n\",\n \"print(\\\"All assertion are successfuly satisfied.\\\")\"\n ],\n \"metadata\": {\n \"id\": \"UM3TB_KxgzSh\"\n },\n \"execution_count\": null,\n \"outputs\": []\n },\n {\n \"cell_type\": \"code\",\n \"source\": [],\n \"metadata\": {\n \"id\": \"xSkwu_ovYaI7\"\n },\n \"execution_count\": null,\n \"outputs\": []\n }\n ]\n}" }, { "path": "jreleaser.yml", "content": "# Generated with JReleaser 1.17.0 at 2025-04-28T13:27:24.943485+01:00\nproject:\n name: neo4j-spark-connector\n description: Neo4j Connector for Spark\n authors:\n - Connectors Team\n license: Apache-2.0\n copyright: Neo4j, Inc.\n links:\n homepage: https://github.com/neo4j/neo4j-spark-connector\n languages:\n java:\n groupId: org.neo4j\n\nrelease:\n github:\n owner: neo4j\n name: neo4j-spark-connector\n tagName: \"{{projectVersion}}\"\n update:\n enabled: true\n sections:\n - TITLE\n - BODY\n - ASSETS\n artifacts: false\n files: true\n checksums: true\n changelog:\n formatted: ALWAYS\n preset: conventional-commits\n skipMergeCommits: true\n format: \"- {{commitShortHash}} {{conventionalCommitDescription}}\"\n links: true\n labelers:\n - title: ci\n label: ci\n excludeLabels:\n - ci\n hide:\n contributors:\n - '[bot]'\n - GitHub\n\nchecksum:\n name: 'neo4j-connector-apache-spark-{{projectVersion}}_checksums.txt'\n algorithms:\n - SHA_256\n files: true\n artifacts: true\n individual: true\n\nfiles:\n globs:\n - pattern: artifacts/*.jar\n - pattern: artifacts/*.zip\n extraProperties:\n skipRelease: \"true\"\n\nassemble:\n archive:\n zip:\n active: ALWAYS\n exported: true\n stereotype: NONE\n archiveName: neo4j-connector-apache-spark-{{projectVersion}}\n distributionType: BINARY\n attachPlatform: false\n formats:\n - ZIP\n fileSets:\n - input: '{{basedir}}/artifacts'\n output: .\n includes:\n - '{{projectName}}-{{projectVersion}}*.jar'\n templateDirectory: spark-3/src/jreleaser/assemblers/zip\n\nhooks:\n script:\n\n before:\n - filter:\n includes: [ \"assemble\" ]\n shell: BASH\n run: |\n rm -rf artifacts\n - filter:\n includes: [ \"assemble\" ]\n matrix:\n vars:\n scala: [ \"2.12\", \"2.13\" ]\n continueOnError: false\n verbose: true\n shell: BASH\n run: |\n mkdir artifacts || true\n ./maven-release.sh deploy {{matrix.scala}} default::file://{{basedir}}/target/{{matrix.scala}}/maven-artifacts\n cp -r {{basedir}}/target/{{matrix.scala}}/maven-artifacts artifacts/\n cp -r {{basedir}}/spark-3/target/{{projectName}}*.zip artifacts/\n cp -r {{basedir}}/spark-3/target/{{projectName}}*.jar artifacts/\n\nsigning:\n active: ALWAYS\n mode: COMMAND\n command:\n homeDir: '~/.gnupg'\n\ndeploy:\n maven:\n active: ALWAYS\n mavenCentral:\n artifacts:\n active: ALWAYS\n url: https://central.sonatype.com/api/v1/publisher\n applyMavenCentralRules: true\n namespace: org.neo4j\n verifyPom: false\n stagingRepositories:\n - ./artifacts/maven-artifacts\n\nannounce:\n slack:\n channels:\n - '#release'\n - '#team-spark'\n message: ':tada: @release-pm {{projectNameCapitalized}} {{projectVersion}} has been released! {{releaseNotesUrl}}'\n" }, { "path": "maven-release.sh", "content": "#!/bin/bash\n\nset -eEuxo pipefail\n\nif [[ $# -lt 2 ]] ; then\n echo \"Usage ./maven-release.sh []\"\n exit 1\nfi\n\nexit_script() {\n echo \"Process terminated cleaning up resources\"\n mv -f pom.xml.bak pom.xml\n mv -f common/pom.xml.bak common/pom.xml\n mv -f test-support/pom.xml.bak test-support/pom.xml\n mv -f spark-3/pom.xml.bak spark-3/pom.xml\n trap - SIGINT SIGTERM # clear the trap\n kill -- -$$ || true # Sends SIGTERM to child/sub processes\n}\n\nmvn_evaluate() {\n local expression\n expression=\"${1}\"\n ./mvnw -B help:evaluate -Dexpression=\"${expression}\" --quiet -DforceStdout\n}\n\ntrap exit_script SIGINT SIGTERM\n\nGOAL=$1\nSCALA_VERSION=$2\nSPARK_VERSION=3\nif [[ $# -eq 3 ]] ; then\n ALT_DEPLOYMENT_REPOSITORY=\"-DaltDeploymentRepository=$3\"\nelse\n ALT_DEPLOYMENT_REPOSITORY=\"\"\nfi\n\ncase $(sed --help 2>&1) in\n *GNU*) sed_i () { sed -i \"$@\"; };;\n *) sed_i () { sed -i '' \"$@\"; };;\nesac\n\n\n\nPROJECT_VERSION=$(mvn_evaluate \"project.version\")\nSPARK_PACKAGES_VERSION=\"${PROJECT_VERSION}-s_$SCALA_VERSION\"\n\n# backup files\ncp pom.xml pom.xml.bak\ncp common/pom.xml common/pom.xml.bak\ncp test-support/pom.xml test-support/pom.xml.bak\ncp spark-3/pom.xml spark-3/pom.xml.bak\n\n./mvnw -B versions:set -DnewVersion=${PROJECT_VERSION}_for_spark_${SPARK_VERSION} -DgenerateBackupPoms=false\n\n# replace pom files with target scala version\nsed_i \"s/neo4j-connector-apache-spark_parent<\\/artifactId>/neo4j-connector-apache-spark_${SCALA_VERSION}_parent<\\/artifactId>/\" pom.xml\nsed_i \"s/neo4j-connector-apache-spark_parent<\\/artifactId>/neo4j-connector-apache-spark_${SCALA_VERSION}_parent<\\/artifactId>/\" \"test-support/pom.xml\"\nsed_i \"s/neo4j-connector-apache-spark_test-support<\\/artifactId>/neo4j-connector-apache-spark_${SCALA_VERSION}_test-support<\\/artifactId>/\" \"test-support/pom.xml\"\n\nsed_i \"s/neo4j-connector-apache-spark_common<\\/artifactId>/neo4j-connector-apache-spark_${SCALA_VERSION}_common<\\/artifactId>/\" \"common/pom.xml\"\nsed_i \"s/neo4j-connector-apache-spark_parent<\\/artifactId>/neo4j-connector-apache-spark_${SCALA_VERSION}_parent<\\/artifactId>/\" \"common/pom.xml\"\nsed_i \"s/neo4j-connector-apache-spark_test-support<\\/artifactId>/neo4j-connector-apache-spark_${SCALA_VERSION}_test-support<\\/artifactId>/\" \"common/pom.xml\"\n\nsed_i \"s/neo4j-connector-apache-spark<\\/artifactId>/neo4j-connector-apache-spark_${SCALA_VERSION}<\\/artifactId>/\" \"spark-3/pom.xml\"\nsed_i \"s/neo4j-connector-apache-spark_parent<\\/artifactId>/neo4j-connector-apache-spark_${SCALA_VERSION}_parent<\\/artifactId>/\" \"spark-3/pom.xml\"\nsed_i \"s/neo4j-connector-apache-spark_common<\\/artifactId>/neo4j-connector-apache-spark_${SCALA_VERSION}_common<\\/artifactId>/\" \"spark-3/pom.xml\"\nsed_i \"s/neo4j-connector-apache-spark_test-support<\\/artifactId>/neo4j-connector-apache-spark_${SCALA_VERSION}_test-support<\\/artifactId>/\" \"spark-3/pom.xml\"\nsed_i \"s//${SPARK_PACKAGES_VERSION}<\\/spark-packages.version>/\" \"spark-3/pom.xml\"\n\n# build\n./mvnw -B clean \"${GOAL}\" -Dscala-\"${SCALA_VERSION}\" -DskipTests ${ALT_DEPLOYMENT_REPOSITORY}\n\nif [ ! ${CI:-false} = true ]; then\n exit_script\nfi\n" }, { "path": "mvnw", "content": "#!/bin/sh\n# ----------------------------------------------------------------------------\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n# ----------------------------------------------------------------------------\n\n# ----------------------------------------------------------------------------\n# Apache Maven Wrapper startup batch script, version 3.3.2\n#\n# Optional ENV vars\n# -----------------\n# JAVA_HOME - location of a JDK home dir, required when download maven via java source\n# MVNW_REPOURL - repo url base for downloading maven distribution\n# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven\n# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output\n# ----------------------------------------------------------------------------\n\nset -euf\n[ \"${MVNW_VERBOSE-}\" != debug ] || set -x\n\n# OS specific support.\nnative_path() { printf %s\\\\n \"$1\"; }\ncase \"$(uname)\" in\nCYGWIN* | MINGW*)\n [ -z \"${JAVA_HOME-}\" ] || JAVA_HOME=\"$(cygpath --unix \"$JAVA_HOME\")\"\n native_path() { cygpath --path --windows \"$1\"; }\n ;;\nesac\n\n# set JAVACMD and JAVACCMD\nset_java_home() {\n # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched\n if [ -n \"${JAVA_HOME-}\" ]; then\n if [ -x \"$JAVA_HOME/jre/sh/java\" ]; then\n # IBM's JDK on AIX uses strange locations for the executables\n JAVACMD=\"$JAVA_HOME/jre/sh/java\"\n JAVACCMD=\"$JAVA_HOME/jre/sh/javac\"\n else\n JAVACMD=\"$JAVA_HOME/bin/java\"\n JAVACCMD=\"$JAVA_HOME/bin/javac\"\n\n if [ ! -x \"$JAVACMD\" ] || [ ! -x \"$JAVACCMD\" ]; then\n echo \"The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run.\" >&2\n echo \"JAVA_HOME is set to \\\"$JAVA_HOME\\\", but \\\"\\$JAVA_HOME/bin/java\\\" or \\\"\\$JAVA_HOME/bin/javac\\\" does not exist.\" >&2\n return 1\n fi\n fi\n else\n JAVACMD=\"$(\n 'set' +e\n 'unset' -f command 2>/dev/null\n 'command' -v java\n )\" || :\n JAVACCMD=\"$(\n 'set' +e\n 'unset' -f command 2>/dev/null\n 'command' -v javac\n )\" || :\n\n if [ ! -x \"${JAVACMD-}\" ] || [ ! -x \"${JAVACCMD-}\" ]; then\n echo \"The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run.\" >&2\n return 1\n fi\n fi\n}\n\n# hash string like Java String::hashCode\nhash_string() {\n str=\"${1:-}\" h=0\n while [ -n \"$str\" ]; do\n char=\"${str%\"${str#?}\"}\"\n h=$(((h * 31 + $(LC_CTYPE=C printf %d \"'$char\")) % 4294967296))\n str=\"${str#?}\"\n done\n printf %x\\\\n $h\n}\n\nverbose() { :; }\n[ \"${MVNW_VERBOSE-}\" != true ] || verbose() { printf %s\\\\n \"${1-}\"; }\n\ndie() {\n printf %s\\\\n \"$1\" >&2\n exit 1\n}\n\ntrim() {\n # MWRAPPER-139:\n # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.\n # Needed for removing poorly interpreted newline sequences when running in more\n # exotic environments such as mingw bash on Windows.\n printf \"%s\" \"${1}\" | tr -d '[:space:]'\n}\n\n# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties\nwhile IFS=\"=\" read -r key value; do\n case \"${key-}\" in\n distributionUrl) distributionUrl=$(trim \"${value-}\") ;;\n distributionSha256Sum) distributionSha256Sum=$(trim \"${value-}\") ;;\n esac\ndone <\"${0%/*}/.mvn/wrapper/maven-wrapper.properties\"\n[ -n \"${distributionUrl-}\" ] || die \"cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties\"\n\ncase \"${distributionUrl##*/}\" in\nmaven-mvnd-*bin.*)\n MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/\n case \"${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)\" in\n *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;\n :Darwin*x86_64) distributionPlatform=darwin-amd64 ;;\n :Darwin*arm64) distributionPlatform=darwin-aarch64 ;;\n :Linux*x86_64*) distributionPlatform=linux-amd64 ;;\n *)\n echo \"Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version\" >&2\n distributionPlatform=linux-amd64\n ;;\n esac\n distributionUrl=\"${distributionUrl%-bin.*}-$distributionPlatform.zip\"\n ;;\nmaven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;\n*) MVN_CMD=\"mvn${0##*/mvnw}\" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;\nesac\n\n# apply MVNW_REPOURL and calculate MAVEN_HOME\n# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/\n[ -z \"${MVNW_REPOURL-}\" ] || distributionUrl=\"$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*\"$_MVNW_REPO_PATTERN\"}\"\ndistributionUrlName=\"${distributionUrl##*/}\"\ndistributionUrlNameMain=\"${distributionUrlName%.*}\"\ndistributionUrlNameMain=\"${distributionUrlNameMain%-bin}\"\nMAVEN_USER_HOME=\"${MAVEN_USER_HOME:-${HOME}/.m2}\"\nMAVEN_HOME=\"${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string \"$distributionUrl\")\"\n\nexec_maven() {\n unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :\n exec \"$MAVEN_HOME/bin/$MVN_CMD\" \"$@\" || die \"cannot exec $MAVEN_HOME/bin/$MVN_CMD\"\n}\n\nif [ -d \"$MAVEN_HOME\" ]; then\n verbose \"found existing MAVEN_HOME at $MAVEN_HOME\"\n exec_maven \"$@\"\nfi\n\ncase \"${distributionUrl-}\" in\n*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;\n*) die \"distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'\" ;;\nesac\n\n# prepare tmp dir\nif TMP_DOWNLOAD_DIR=\"$(mktemp -d)\" && [ -d \"$TMP_DOWNLOAD_DIR\" ]; then\n clean() { rm -rf -- \"$TMP_DOWNLOAD_DIR\"; }\n trap clean HUP INT TERM EXIT\nelse\n die \"cannot create temp dir\"\nfi\n\nmkdir -p -- \"${MAVEN_HOME%/*}\"\n\n# Download and Install Apache Maven\nverbose \"Couldn't find MAVEN_HOME, downloading and installing it ...\"\nverbose \"Downloading from: $distributionUrl\"\nverbose \"Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName\"\n\n# select .zip or .tar.gz\nif ! command -v unzip >/dev/null; then\n distributionUrl=\"${distributionUrl%.zip}.tar.gz\"\n distributionUrlName=\"${distributionUrl##*/}\"\nfi\n\n# verbose opt\n__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''\n[ \"${MVNW_VERBOSE-}\" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v\n\n# normalize http auth\ncase \"${MVNW_PASSWORD:+has-password}\" in\n'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;\nhas-password) [ -n \"${MVNW_USERNAME-}\" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;\nesac\n\nif [ -z \"${MVNW_USERNAME-}\" ] && command -v wget >/dev/null; then\n verbose \"Found wget ... using wget\"\n wget ${__MVNW_QUIET_WGET:+\"$__MVNW_QUIET_WGET\"} \"$distributionUrl\" -O \"$TMP_DOWNLOAD_DIR/$distributionUrlName\" || die \"wget: Failed to fetch $distributionUrl\"\nelif [ -z \"${MVNW_USERNAME-}\" ] && command -v curl >/dev/null; then\n verbose \"Found curl ... using curl\"\n curl ${__MVNW_QUIET_CURL:+\"$__MVNW_QUIET_CURL\"} -f -L -o \"$TMP_DOWNLOAD_DIR/$distributionUrlName\" \"$distributionUrl\" || die \"curl: Failed to fetch $distributionUrl\"\nelif set_java_home; then\n verbose \"Falling back to use Java to download\"\n javaSource=\"$TMP_DOWNLOAD_DIR/Downloader.java\"\n targetZip=\"$TMP_DOWNLOAD_DIR/$distributionUrlName\"\n cat >\"$javaSource\" <<-END\n\tpublic class Downloader extends java.net.Authenticator\n\t{\n\t protected java.net.PasswordAuthentication getPasswordAuthentication()\n\t {\n\t return new java.net.PasswordAuthentication( System.getenv( \"MVNW_USERNAME\" ), System.getenv( \"MVNW_PASSWORD\" ).toCharArray() );\n\t }\n\t public static void main( String[] args ) throws Exception\n\t {\n\t setDefault( new Downloader() );\n\t java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );\n\t }\n\t}\n\tEND\n # For Cygwin/MinGW, switch paths to Windows format before running javac and java\n verbose \" - Compiling Downloader.java ...\"\n \"$(native_path \"$JAVACCMD\")\" \"$(native_path \"$javaSource\")\" || die \"Failed to compile Downloader.java\"\n verbose \" - Running Downloader.java ...\"\n \"$(native_path \"$JAVACMD\")\" -cp \"$(native_path \"$TMP_DOWNLOAD_DIR\")\" Downloader \"$distributionUrl\" \"$(native_path \"$targetZip\")\"\nfi\n\n# If specified, validate the SHA-256 sum of the Maven distribution zip file\nif [ -n \"${distributionSha256Sum-}\" ]; then\n distributionSha256Result=false\n if [ \"$MVN_CMD\" = mvnd.sh ]; then\n echo \"Checksum validation is not supported for maven-mvnd.\" >&2\n echo \"Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties.\" >&2\n exit 1\n elif command -v sha256sum >/dev/null; then\n if echo \"$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName\" | sha256sum -c >/dev/null 2>&1; then\n distributionSha256Result=true\n fi\n elif command -v shasum >/dev/null; then\n if echo \"$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName\" | shasum -a 256 -c >/dev/null 2>&1; then\n distributionSha256Result=true\n fi\n else\n echo \"Checksum validation was requested but neither 'sha256sum' or 'shasum' are available.\" >&2\n echo \"Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties.\" >&2\n exit 1\n fi\n if [ $distributionSha256Result = false ]; then\n echo \"Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised.\" >&2\n echo \"If you updated your Maven version, you need to update the specified distributionSha256Sum property.\" >&2\n exit 1\n fi\nfi\n\n# unzip and move\nif command -v unzip >/dev/null; then\n unzip ${__MVNW_QUIET_UNZIP:+\"$__MVNW_QUIET_UNZIP\"} \"$TMP_DOWNLOAD_DIR/$distributionUrlName\" -d \"$TMP_DOWNLOAD_DIR\" || die \"failed to unzip\"\nelse\n tar xzf${__MVNW_QUIET_TAR:+\"$__MVNW_QUIET_TAR\"} \"$TMP_DOWNLOAD_DIR/$distributionUrlName\" -C \"$TMP_DOWNLOAD_DIR\" || die \"failed to untar\"\nfi\nprintf %s\\\\n \"$distributionUrl\" >\"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url\"\nmv -- \"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain\" \"$MAVEN_HOME\" || [ -d \"$MAVEN_HOME\" ] || die \"fail to move MAVEN_HOME\"\n\nclean || :\nexec_maven \"$@\"\n" }, { "path": "mvnw.cmd", "content": "<# : batch portion\r\n@REM ----------------------------------------------------------------------------\r\n@REM Licensed to the Apache Software Foundation (ASF) under one\r\n@REM or more contributor license agreements. See the NOTICE file\r\n@REM distributed with this work for additional information\r\n@REM regarding copyright ownership. The ASF licenses this file\r\n@REM to you under the Apache License, Version 2.0 (the\r\n@REM \"License\"); you may not use this file except in compliance\r\n@REM with the License. You may obtain a copy of the License at\r\n@REM\r\n@REM http://www.apache.org/licenses/LICENSE-2.0\r\n@REM\r\n@REM Unless required by applicable law or agreed to in writing,\r\n@REM software distributed under the License is distributed on an\r\n@REM \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n@REM KIND, either express or implied. See the License for the\r\n@REM specific language governing permissions and limitations\r\n@REM under the License.\r\n@REM ----------------------------------------------------------------------------\r\n\r\n@REM ----------------------------------------------------------------------------\r\n@REM Apache Maven Wrapper startup batch script, version 3.3.2\r\n@REM\r\n@REM Optional ENV vars\r\n@REM MVNW_REPOURL - repo url base for downloading maven distribution\r\n@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven\r\n@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output\r\n@REM ----------------------------------------------------------------------------\r\n\r\n@IF \"%__MVNW_ARG0_NAME__%\"==\"\" (SET __MVNW_ARG0_NAME__=%~nx0)\r\n@SET __MVNW_CMD__=\r\n@SET __MVNW_ERROR__=\r\n@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%\r\n@SET PSModulePath=\r\n@FOR /F \"usebackq tokens=1* delims==\" %%A IN (`powershell -noprofile \"& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}\"`) DO @(\r\n IF \"%%A\"==\"MVN_CMD\" (set __MVNW_CMD__=%%B) ELSE IF \"%%B\"==\"\" (echo %%A) ELSE (echo %%A=%%B)\r\n)\r\n@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%\r\n@SET __MVNW_PSMODULEP_SAVE=\r\n@SET __MVNW_ARG0_NAME__=\r\n@SET MVNW_USERNAME=\r\n@SET MVNW_PASSWORD=\r\n@IF NOT \"%__MVNW_CMD__%\"==\"\" (%__MVNW_CMD__% %*)\r\n@echo Cannot start maven from wrapper >&2 && exit /b 1\r\n@GOTO :EOF\r\n: end batch / begin powershell #>\r\n\r\n$ErrorActionPreference = \"Stop\"\r\nif ($env:MVNW_VERBOSE -eq \"true\") {\r\n $VerbosePreference = \"Continue\"\r\n}\r\n\r\n# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties\r\n$distributionUrl = (Get-Content -Raw \"$scriptDir/.mvn/wrapper/maven-wrapper.properties\" | ConvertFrom-StringData).distributionUrl\r\nif (!$distributionUrl) {\r\n Write-Error \"cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties\"\r\n}\r\n\r\nswitch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {\r\n \"maven-mvnd-*\" {\r\n $USE_MVND = $true\r\n $distributionUrl = $distributionUrl -replace '-bin\\.[^.]*$',\"-windows-amd64.zip\"\r\n $MVN_CMD = \"mvnd.cmd\"\r\n break\r\n }\r\n default {\r\n $USE_MVND = $false\r\n $MVN_CMD = $script -replace '^mvnw','mvn'\r\n break\r\n }\r\n}\r\n\r\n# apply MVNW_REPOURL and calculate MAVEN_HOME\r\n# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/\r\nif ($env:MVNW_REPOURL) {\r\n $MVNW_REPO_PATTERN = if ($USE_MVND) { \"/org/apache/maven/\" } else { \"/maven/mvnd/\" }\r\n $distributionUrl = \"$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')\"\r\n}\r\n$distributionUrlName = $distributionUrl -replace '^.*/',''\r\n$distributionUrlNameMain = $distributionUrlName -replace '\\.[^.]*$','' -replace '-bin$',''\r\n$MAVEN_HOME_PARENT = \"$HOME/.m2/wrapper/dists/$distributionUrlNameMain\"\r\nif ($env:MAVEN_USER_HOME) {\r\n $MAVEN_HOME_PARENT = \"$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain\"\r\n}\r\n$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString(\"x2\")}) -join ''\r\n$MAVEN_HOME = \"$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME\"\r\n\r\nif (Test-Path -Path \"$MAVEN_HOME\" -PathType Container) {\r\n Write-Verbose \"found existing MAVEN_HOME at $MAVEN_HOME\"\r\n Write-Output \"MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD\"\r\n exit $?\r\n}\r\n\r\nif (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {\r\n Write-Error \"distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl\"\r\n}\r\n\r\n# prepare tmp dir\r\n$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile\r\n$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path \"$TMP_DOWNLOAD_DIR_HOLDER.dir\"\r\n$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null\r\ntrap {\r\n if ($TMP_DOWNLOAD_DIR.Exists) {\r\n try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }\r\n catch { Write-Warning \"Cannot remove $TMP_DOWNLOAD_DIR\" }\r\n }\r\n}\r\n\r\nNew-Item -Itemtype Directory -Path \"$MAVEN_HOME_PARENT\" -Force | Out-Null\r\n\r\n# Download and Install Apache Maven\r\nWrite-Verbose \"Couldn't find MAVEN_HOME, downloading and installing it ...\"\r\nWrite-Verbose \"Downloading from: $distributionUrl\"\r\nWrite-Verbose \"Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName\"\r\n\r\n$webclient = New-Object System.Net.WebClient\r\nif ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {\r\n $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)\r\n}\r\n[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12\r\n$webclient.DownloadFile($distributionUrl, \"$TMP_DOWNLOAD_DIR/$distributionUrlName\") | Out-Null\r\n\r\n# If specified, validate the SHA-256 sum of the Maven distribution zip file\r\n$distributionSha256Sum = (Get-Content -Raw \"$scriptDir/.mvn/wrapper/maven-wrapper.properties\" | ConvertFrom-StringData).distributionSha256Sum\r\nif ($distributionSha256Sum) {\r\n if ($USE_MVND) {\r\n Write-Error \"Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties.\"\r\n }\r\n Import-Module $PSHOME\\Modules\\Microsoft.PowerShell.Utility -Function Get-FileHash\r\n if ((Get-FileHash \"$TMP_DOWNLOAD_DIR/$distributionUrlName\" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {\r\n Write-Error \"Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property.\"\r\n }\r\n}\r\n\r\n# unzip and move\r\nExpand-Archive \"$TMP_DOWNLOAD_DIR/$distributionUrlName\" -DestinationPath \"$TMP_DOWNLOAD_DIR\" | Out-Null\r\nRename-Item -Path \"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain\" -NewName $MAVEN_HOME_NAME | Out-Null\r\ntry {\r\n Move-Item -Path \"$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME\" -Destination $MAVEN_HOME_PARENT | Out-Null\r\n} catch {\r\n if (! (Test-Path -Path \"$MAVEN_HOME\" -PathType Container)) {\r\n Write-Error \"fail to move MAVEN_HOME\"\r\n }\r\n} finally {\r\n try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }\r\n catch { Write-Warning \"Cannot remove $TMP_DOWNLOAD_DIR\" }\r\n}\r\n\r\nWrite-Output \"MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD\"\r\n" }, { "path": "package.json", "content": "{\n \"devDependencies\": {\n \"@commitlint/cli\": \"^20.4.2\",\n \"@commitlint/config-conventional\": \"^20.4.2\",\n \"@commitlint/lint\": \"^20.4.2\",\n \"@commitlint/load\": \"^20.4.0\",\n \"danger\": \"^13.0.7\",\n \"husky\": \"^9.1.7\"\n },\n \"scripts\": {\n \"prepare\": \"husky\"\n }\n}\n" }, { "path": "pom.xml", "content": "\n\n 4.0.0\n org.neo4j\n neo4j-connector-apache-spark_parent\n 5.4.3-SNAPSHOT\n pom\n neo4j-connector-apache-spark_parent\n Neo4j Connector for Apache Spark using the binary Bolt Driver\n https://github.com/neo4j/neo4j-spark-connector\n \n Neo4j, Inc.\n https://neo4j.com/\n \n \n \n Apache License, Version 2.0\n https://www.apache.org/licenses/LICENSE-2.0.txt\n manual\n \n \n \n \n team-connectors\n Connectors Team\n Neo4j\n https://neo4j.com\n \n \n \n common\n test-support\n spark-3\n \n \n https://github.com/neo4j/neo4j-spark-connector\n \n \n 2026.02.5\n 1.3.3\n 1.28.0\n 3.20.0\n 1.0.0-rc2\n 2022.11.0\n 4.4.22\n 1.8\n \n 3.4.3.Final\n 4.13.2\n 4.6\n 1.7.12\n /license/neo4j_apache_v2/notice.txt\n 3.5.0\n 3.15.0\n 3.10.0\n 3.1.4\n 3.6.2\n 3.5.5\n 3.1.4\n 3.5.0\n 3.5.0\n 3.21.0\n 3.4.0\n 3.5.5\n false\n 4.4.20\n 4.1.132.Final\n UTF-8\n 4.9.10\n 2.12\n 2.12.20\n 2.0.17\n 4.0.0\n 3.5.8\n 3.4.0\n \n 2.0.4\n \n \n \n \n io.netty\n netty-bom\n ${netty-bom.version}\n pom\n import\n \n \n org.testcontainers\n testcontainers-bom\n ${testcontainers.version}\n pom\n import\n \n \n junit\n junit\n ${junit.version}\n \n \n org.apache.commons\n commons-compress\n ${commons-compress.version}\n \n \n org.apache.commons\n commons-lang3\n ${commons-lang3.version}\n \n \n org.apache.spark\n spark-sql_${scala.binary.version}\n ${spark.version}\n \n \n org.hamcrest\n hamcrest\n 3.0\n \n \n org.jboss.logging\n jboss-logging\n ${jboss-logging.version}\n \n \n org.neo4j\n caniuse-core\n ${caniuse.version}\n \n \n org.neo4j\n caniuse-neo4j-detection\n ${caniuse.version}\n \n \n org.neo4j\n neo4j-cypher-dsl\n ${cypherdsl.version}\n \n \n org.neo4j.connectors\n commons-authn-keycloak\n ${connectors-commons.version}\n \n \n org.neo4j.connectors\n commons-authn-provided\n ${connectors-commons.version}\n \n \n org.neo4j.connectors\n commons-authn-spi\n ${connectors-commons.version}\n \n \n org.neo4j.connectors\n commons-reauth-driver\n ${connectors-commons.version}\n \n \n org.neo4j.driver\n neo4j-java-driver-slim\n ${driver.version}\n \n \n org.scala-lang\n scala-library\n ${scala.version}\n \n \n org.scala-lang\n scala-reflect\n ${scala.version}\n \n \n org.scalatest\n scalatest_${scala.binary.version}\n 3.2.20\n \n \n org.scalatestplus\n junit-4-13_${scala.binary.version}\n 3.2.20.0\n \n \n org.slf4j\n slf4j-api\n ${slf4j-api.version}\n \n \n org.apache.spark\n spark-core_${scala.binary.version}\n ${spark.version}\n provided\n \n \n org.apache.xbean\n xbean-asm6-shaded\n \n \n \n \n \n \n \n \n \n org.codehaus.mojo\n versions-maven-plugin\n 2.21.0\n \n false\n \n \n \n org.apache.maven.plugins\n maven-clean-plugin\n ${maven-clean-plugin.version}\n \n \n org.apache.maven.plugins\n maven-deploy-plugin\n ${maven-deploy-plugin.version}\n \n \n org.apache.maven.plugins\n maven-install-plugin\n ${maven-install-plugin.version}\n \n \n org.apache.maven.plugins\n maven-site-plugin\n ${maven-site-plugin.version}\n \n \n org.apache.maven.plugins\n maven-jar-plugin\n ${maven-jar-plugin.version}\n \n \n \n true\n \n \n \n \n \n org.apache.maven.plugins\n maven-resources-plugin\n ${maven-resources-plugin.version}\n \n ${project.build.sourceEncoding}\n \n \n \n org.neo4j.build.plugins\n licensing-maven-plugin\n ${licensing-maven-plugin.version}\n \n true\n true\n true\n ${licensing.prepend.text}\n ^((org.neo4j){1}|(org.neo4j.connectors){1}|(org.neo4j.connectors.kafka){1}|(org.neo4j.driver){1})$\n compile,runtime\n \n \n \n org.neo4j.build\n resources\n ${build-resources.version}\n \n \n \n \n list-all-licenses\n \n check\n \n compile\n \n \n \n licensing/licensing-requirements-base.xml\n \n ${project.artifactId}-${project.version}-NOTICE.txt\n ${project.build.directory}/../NOTICE.txt\n /licensing/list-prefix.txt\n ${project.artifactId}-${project.version}-LICENSES.txt\n ${project.build.directory}/../LICENSES.txt\n \n \n \n \n \n org.apache.maven.plugins\n maven-source-plugin\n ${maven-source-plugin.version}\n \n \n attach-sources\n \n jar\n \n \n \n \n \n org.apache.maven.plugins\n maven-enforcer-plugin\n ${maven-enforcer-plugin.version}\n \n \n enforce\n \n enforce\n \n validate\n \n \n \n [3.6.3,)\n \n \n 8\n \n \n \n \n \n \n \n \n org.apache.maven.plugins\n maven-compiler-plugin\n ${maven-compiler-plugin.version}\n \n ${java.version}\n ${java.version}\n \n \n \n org.apache.maven.plugins\n maven-failsafe-plugin\n ${maven-failsafe-plugin.version}\n \n \n **/*IT.*\n **/*TSE.*\n \n 1\n false\n ${surefire.jvm.args}\n false\n \n \n \n \n integration-test\n verify\n \n integration-test\n \n \n \n \n org.apache.maven.plugins\n maven-dependency-plugin\n ${maven-dependency-plugin.version}\n \n \n com.mycila\n license-maven-plugin\n ${license-maven-plugin.version}\n \n true\n \n \n
/license/neo4j_apache_v2/header.txt
\n \n src/**/*.scala\n src/**/*.java\n \n \n \n \n SLASHSTAR_STYLE\n \n \n \n \n org.neo4j.build\n resources\n ${build-resources.version}\n \n \n \n \n check-licenses\n \n check\n \n compile\n \n \n \n \n org.apache.maven.plugins\n maven-surefire-plugin\n ${maven-surefire-plugin.version}\n \n ${surefire.jvm.args}\n \n \n \n net.alchim31.maven\n scala-maven-plugin\n ${scala-maven-plugin.version}\n \n ${scala.version}\n ${scala.binary.version}\n \n -target:jvm-1.8\n \n \n -Xms64m\n -Xmx1024m\n \n \n \n \n scala-compile\n \n add-source\n compile\n testCompile\n doc-jar\n \n process-resources\n \n \n scala-test-compile\n \n testCompile\n \n test-compile\n \n \n \n \n maven-assembly-plugin\n 3.8.0\n \n \n jar-with-dependencies\n \n ${project.artifactId}-${project.version}\n false\n \n \n \n \n single\n \n package\n \n \n \n \n com.github.ekryd.sortpom\n sortpom-maven-plugin\n ${sortpom-maven-plugin.version}\n \n ${project.build.sourceEncoding}\n false\n schemaLocation\n 4\n true\n scope,groupId,artifactId\n false\n false\n \n \n \n \n verify\n \n validate\n \n STOP\n \n \n \n \n \n com.diffplug.spotless\n spotless-maven-plugin\n ${spotless-maven-plugin.version}\n \n \n \n \n src/main/scala/**/*.scala\n src/test/scala/**/*.scala\n \n \n 3.8.4-RC3\n scalafmt/scalafmt.conf\n 2.12\n \n \n \n \n \n org.neo4j.build\n resources\n ${build-resources.version}\n \n \n \n \n \n check\n \n compile\n \n \n \n \n \n \n \n org.apache.maven.plugins\n maven-enforcer-plugin\n \n \n \n \n \n scala-2.13\n \n \n scala-2.13\n \n \n \n 2.13\n 2.13.16\n \n \n \n \n \n \n has-sources\n \n false\n \n src\n \n \n \n \n \n org.neo4j.build.plugins\n licensing-maven-plugin\n \n \n com.mycila\n license-maven-plugin\n \n \n org.apache.maven.plugins\n maven-source-plugin\n \n \n \n \n \n enable-jdk11-plugins\n \n [11,)\n \n \n -XX:+IgnoreUnrecognizedVMOptions\n --add-opens=java.base/java.lang=ALL-UNNAMED\n --add-opens=java.base/java.lang.invoke=ALL-UNNAMED\n --add-opens=java.base/java.lang.reflect=ALL-UNNAMED\n --add-opens=java.base/java.io=ALL-UNNAMED\n --add-opens=java.base/java.net=ALL-UNNAMED\n --add-opens=java.base/java.nio=ALL-UNNAMED\n --add-opens=java.base/java.util=ALL-UNNAMED\n --add-opens=java.base/java.util.concurrent=ALL-UNNAMED\n --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED\n --add-opens=java.base/sun.nio.ch=ALL-UNNAMED\n --add-opens=java.base/sun.nio.cs=ALL-UNNAMED\n --add-opens=java.base/sun.security.action=ALL-UNNAMED\n --add-opens=java.base/sun.util.calendar=ALL-UNNAMED\n --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED\n -Djdk.reflect.useDirectMethodHandle=false\n \n \n \n \n com.github.ekryd.sortpom\n sortpom-maven-plugin\n \n \n \n \n \n enable-jdk17-plugins\n \n [17,)\n \n \n \n \n com.diffplug.spotless\n spotless-maven-plugin\n \n \n \n \n \n\n" }, { "path": "scripts/python/requirements.txt", "content": "pyspark==3.5.5\ntestcontainers[neo4j]\nsix\ntzlocal==2.1" }, { "path": "scripts/python/test_spark.py", "content": "#!/usr/bin/env python3\n\nimport unittest\nimport sys\nimport datetime\n\nfrom tzlocal import get_localzone\nfrom testcontainers.neo4j import Neo4jContainer\nfrom pyspark.sql import SparkSession\nfrom neo4j import Driver\n\n\nclass SparkTest(unittest.TestCase):\n neo4j_driver: Driver = None\n neo4j_container: Neo4jContainer = None\n spark: SparkSession = None\n\n def tearDown(self):\n with self.neo4j_driver.session(database=\"system\") as session:\n session.run(\"CREATE OR REPLACE DATABASE neo4j WAIT 30 seconds\").consume()\n\n def init_test(self, query, parameters=None):\n with self.neo4j_driver.session() as session:\n session.run(query, parameters).consume()\n\n return (\n self.spark.read.format(\"org.neo4j.spark.DataSource\")\n .option(\"url\", self.neo4j_container.get_connection_url())\n .option(\"authentication.type\", \"basic\")\n .option(\"authentication.basic.username\", \"neo4j\")\n .option(\"authentication.basic.password\", \"password\")\n .option(\"labels\", \"Person\")\n .load()\n )\n\n def test_string(self):\n name = \"Foobar\"\n df = self.init_test(\"CREATE (p:Person {name: '\" + name + \"'})\")\n\n assert name == df.select(\"name\").collect()[0].name\n\n def test_int(self):\n age = 32\n df = self.init_test(\"CREATE (p:Person {age: \" + str(age) + \"})\")\n\n assert age == df.select(\"age\").collect()[0].age\n\n def test_double(self):\n score = 32.3\n df = self.init_test(\"CREATE (p:Person {score: \" + str(score) + \"})\")\n\n assert score == df.select(\"score\").collect()[0].score\n\n def test_boolean(self):\n df = self.init_test(\"CREATE (p:Person {boolean: true})\")\n\n assert True == df.select(\"boolean\").collect()[0].boolean\n\n def test_time(self):\n time = datetime.time(12, 23, 0, 0, get_localzone())\n df = self.init_test(\n \"CREATE (p:Person {myTime: time({hour:12, minute: 23, second: 0})})\"\n )\n\n timeResult = df.select(\"myTime\").collect()[0].myTime\n\n assert \"offset-time\" == timeResult.type\n # .replace used in case of UTC timezone because of https://stackoverflow.com/a/42777551/1409772\n assert str(time).replace(\"+00:00\", \"Z\") == timeResult.value.split(\"+\")[0]\n\n def test_datetime(self):\n dtString = \"2015-06-24T12:50:35\"\n df = self.init_test(\n \"CREATE (p:Person {datetime: datetime('\" + dtString + \"')})\"\n )\n\n dt = datetime.datetime(2015, 6, 24, 12, 50, 35, 0)\n dtResult = df.select(\"datetime\").collect()[0].datetime\n\n assert dt == dtResult\n\n def test_date(self):\n df = self.init_test(\"CREATE (p:Person {born: date('2009-10-10')})\")\n\n dt = datetime.date(2009, 10, 10)\n dtResult = df.select(\"born\").collect()[0].born\n\n assert dt == dtResult\n\n def test_point(self):\n df = self.init_test(\"CREATE (p:Person {location: point({x: 12.12, y: 13.13})})\")\n\n pointResult = df.select(\"location\").collect()[0].location\n assert \"point-2d\" == pointResult[0]\n assert 7203 == pointResult[1]\n assert 12.12 == pointResult[2]\n assert 13.13 == pointResult[3]\n\n def test_point3d(self):\n df = self.init_test(\n \"CREATE (p:Person {location: point({x: 12.12, y: 13.13, z: 1})})\"\n )\n\n pointResult = df.select(\"location\").collect()[0].location\n assert \"point-3d\" == pointResult[0]\n assert 9157 == pointResult[1]\n assert 12.12 == pointResult[2]\n assert 13.13 == pointResult[3]\n assert 1.0 == pointResult[4]\n\n def test_geopoint(self):\n df = self.init_test(\n \"CREATE (p:Person {location: point({longitude: 12.12, latitude: 13.13})})\"\n )\n\n pointResult = df.select(\"location\").collect()[0].location\n assert \"point-2d\" == pointResult[0]\n assert 4326 == pointResult[1]\n assert 12.12 == pointResult[2]\n assert 13.13 == pointResult[3]\n\n def test_duration(self):\n df = self.init_test(\n \"CREATE (p:Person {range: duration({days: 14, hours:16, minutes: 12})})\"\n )\n\n durationResult = df.select(\"range\").collect()[0].range\n assert \"duration\" == durationResult[0]\n assert 0 == durationResult[1]\n assert 14 == durationResult[2]\n assert 58320 == durationResult[3]\n assert 0 == durationResult[4]\n\n def test_binary(self):\n byte_array = b\"binaries are byte arrays\"\n df = self.init_test(\"CREATE (p:Person {bin: $bytes})\", {\"bytes\": byte_array})\n\n assert byte_array == df.select(\"bin\").collect()[0].bin\n\n def test_string_array(self):\n df = self.init_test(\"CREATE (p:Person {names: ['John', 'Doe']})\")\n\n result = df.select(\"names\").collect()[0].names\n assert \"John\" == result[0]\n assert \"Doe\" == result[1]\n\n def test_int_array(self):\n df = self.init_test(\"CREATE (p:Person {ages: [24, 56]})\")\n\n result = df.select(\"ages\").collect()[0].ages\n assert 24 == result[0]\n assert 56 == result[1]\n\n def test_double_array(self):\n df = self.init_test(\"CREATE (p:Person {scores: [24.11, 56.11]})\")\n\n result = df.select(\"scores\").collect()[0].scores\n assert 24.11 == result[0]\n assert 56.11 == result[1]\n\n def test_boolean_array(self):\n df = self.init_test(\"CREATE (p:Person {field: [true, false]})\")\n\n result = df.select(\"field\").collect()[0].field\n assert True == result[0]\n assert False == result[1]\n\n def test_time_array(self):\n df = self.init_test(\n \"CREATE (p:Person {result: [time({hour:11, minute: 23, second: 0}), time({hour:12, minute: 23, second: 0})]})\"\n )\n\n timeResult = df.select(\"result\").collect()[0].result\n\n # .replace used in case of UTC timezone because of https://stackoverflow.com/a/42777551/1409772\n assert \"offset-time\" == timeResult[0].type\n assert (\n str(datetime.time(11, 23, 0, 0, get_localzone())).replace(\"+00:00\", \"Z\")\n == timeResult[0].value.split(\"+\")[0]\n )\n\n # .replace used in case of UTC timezone because of https://stackoverflow.com/a/42777551/1409772\n assert \"offset-time\" == timeResult[1].type\n assert (\n str(datetime.time(12, 23, 0, 0, get_localzone())).replace(\"+00:00\", \"Z\")\n == timeResult[1].value.split(\"+\")[0]\n )\n\n def test_datetime_array(self):\n df = self.init_test(\n \"CREATE (p:Person {result: [datetime('2007-12-03T10:15:30'), datetime('2008-12-03T10:15:30')]})\"\n )\n\n dt1 = datetime.datetime(2007, 12, 3, 10, 15, 30, 0)\n dt2 = datetime.datetime(2008, 12, 3, 10, 15, 30, 0)\n dtResult = df.select(\"result\").collect()[0].result\n\n assert dt1 == dtResult[0]\n assert dt2 == dtResult[1]\n\n def test_date_array(self):\n df = self.init_test(\n \"CREATE (p:Person {result: [date('2009-10-10'), date('2008-10-10')]})\"\n )\n\n dt1 = datetime.date(2009, 10, 10)\n dt2 = datetime.date(2008, 10, 10)\n dtResult = df.select(\"result\").collect()[0].result\n\n assert dt1 == dtResult[0]\n assert dt2 == dtResult[1]\n\n def test_point_array(self):\n df = self.init_test(\n \"CREATE (p:Person {location: [point({x: 12.12, y: 13.13}), point({x: 13.13, y: 14.14})]})\"\n )\n\n pointResult = df.select(\"location\").collect()[0].location\n assert \"point-2d\" == pointResult[0][0]\n assert 7203 == pointResult[0][1]\n assert 12.12 == pointResult[0][2]\n assert 13.13 == pointResult[0][3]\n\n assert \"point-2d\" == pointResult[1][0]\n assert 7203 == pointResult[1][1]\n assert 13.13 == pointResult[1][2]\n assert 14.14 == pointResult[1][3]\n\n def test_point3d_array(self):\n df = self.init_test(\n \"CREATE (p:Person {location: [point({x: 12.12, y: 13.13, z: 1}), point({x: 14.14, y: 15.15, z: 1})]})\"\n )\n\n pointResult = df.select(\"location\").collect()[0].location\n assert \"point-3d\" == pointResult[0][0]\n assert 9157 == pointResult[0][1]\n assert 12.12 == pointResult[0][2]\n assert 13.13 == pointResult[0][3]\n assert 1.0 == pointResult[0][4]\n\n assert \"point-3d\" == pointResult[1][0]\n assert 9157 == pointResult[1][1]\n assert 14.14 == pointResult[1][2]\n assert 15.15 == pointResult[1][3]\n assert 1.0 == pointResult[1][4]\n\n def test_geopoint_array(self):\n df = self.init_test(\n \"CREATE (p:Person {location: [point({longitude: 12.12, latitude: 13.13}), point({longitude: 14.14, latitude: 15.15})]})\"\n )\n\n pointResult = df.select(\"location\").collect()[0].location\n assert \"point-2d\" == pointResult[0][0]\n assert 4326 == pointResult[0][1]\n assert 12.12 == pointResult[0][2]\n assert 13.13 == pointResult[0][3]\n\n assert \"point-2d\" == pointResult[1][0]\n assert 4326 == pointResult[1][1]\n assert 14.14 == pointResult[1][2]\n assert 15.15 == pointResult[1][3]\n\n def test_duration_array(self):\n df = self.init_test(\n \"CREATE (p:Person {range: [duration({days: 14, hours:16, minutes: 12}), duration({days: 15, hours:16, minutes: 12})]})\"\n )\n\n durationResult = df.select(\"range\").collect()[0].range\n assert \"duration\" == durationResult[0][0]\n assert 0 == durationResult[0][1]\n assert 14 == durationResult[0][2]\n assert 58320 == durationResult[0][3]\n assert 0 == durationResult[0][4]\n\n assert \"duration\" == durationResult[1][0]\n assert 0 == durationResult[1][1]\n assert 15 == durationResult[1][2]\n assert 58320 == durationResult[1][3]\n assert 0 == durationResult[1][4]\n\n def test_unexisting_property(self):\n (\n self.spark.read.format(\"org.neo4j.spark.DataSource\")\n .option(\"url\", self.neo4j_container.get_connection_url())\n .option(\"authentication.type\", \"basic\")\n .option(\"authentication.basic.username\", \"neo4j\")\n .option(\"authentication.basic.password\", \"password\")\n .option(\"relationship.properties\", None)\n .option(\"relationship\", \"FOO\")\n .option(\"relationship.source.labels\", \":Foo\")\n .option(\"relationship.target.labels\", \":Bar\")\n .load()\n )\n # In this case we just test that the job has been executed without any exception\n\n def test_gds(self):\n with self.neo4j_driver.session() as session:\n session.run(\n \"\"\"\n CREATE\n (home:Page {name:'Home'}),\n (about:Page {name:'About'}),\n (product:Page {name:'Product'}),\n (links:Page {name:'Links'}),\n (a:Page {name:'Site A'}),\n (b:Page {name:'Site B'}),\n (c:Page {name:'Site C'}),\n (d:Page {name:'Site D'}),\n\n (home)-[:LINKS {weight: 0.2}]->(about),\n (home)-[:LINKS {weight: 0.2}]->(links),\n (home)-[:LINKS {weight: 0.6}]->(product),\n (about)-[:LINKS {weight: 1.0}]->(home),\n (product)-[:LINKS {weight: 1.0}]->(home),\n (a)-[:LINKS {weight: 1.0}]->(home),\n (b)-[:LINKS {weight: 1.0}]->(home),\n (c)-[:LINKS {weight: 1.0}]->(home),\n (d)-[:LINKS {weight: 1.0}]->(home),\n (links)-[:LINKS {weight: 0.8}]->(home),\n (links)-[:LINKS {weight: 0.05}]->(a),\n (links)-[:LINKS {weight: 0.05}]->(b),\n (links)-[:LINKS {weight: 0.05}]->(c),\n (links)-[:LINKS {weight: 0.05}]->(d);\n \"\"\"\n )\n\n (\n self.spark.read.format(\"org.neo4j.spark.DataSource\")\n .option(\"url\", self.neo4j_container.get_connection_url())\n .option(\"authentication.type\", \"basic\")\n .option(\"authentication.basic.username\", \"neo4j\")\n .option(\"authentication.basic.password\", \"password\")\n .option(\"gds\", \"gds.graph.project\")\n .option(\"gds.graphName\", \"myGraph\")\n .option(\"gds.nodeProjection\", \"Page\")\n .option(\"gds.relationshipProjection\", \"LINKS\")\n .option(\"gds.configuration.relationshipProperties\", \"weight\")\n .load()\n .show(truncate=False)\n )\n\n df = (\n self.spark.read.format(\"org.neo4j.spark.DataSource\")\n .option(\"url\", self.neo4j_container.get_connection_url())\n .option(\"authentication.type\", \"basic\")\n .option(\"authentication.basic.username\", \"neo4j\")\n .option(\"authentication.basic.password\", \"password\")\n .option(\"gds\", \"gds.pageRank.stream\")\n .option(\"gds.graphName\", \"myGraph\")\n .option(\"gds.configuration.concurrency\", \"2\")\n .load()\n )\n\n assert 8 == df.count()\n\n\nif len(sys.argv) != 3:\n print(\"Wrong arguments count\")\n print(sys.argv)\n sys.exit(1)\n\nneo4j_image = str(sys.argv.pop())\nconnector_jar = str(sys.argv.pop())\ncurrent_time_zone = get_localzone().zone\n\nif __name__ == \"__main__\":\n with (\n Neo4jContainer(neo4j_image)\n .with_env(\"NEO4J_ACCEPT_LICENSE_AGREEMENT\", \"yes\")\n .with_env(\"NEO4J_db_temporal_timezone\", current_time_zone)\n .with_env(\"NEO4JLABS_PLUGINS\", '[\"graph-data-science\"]')\n ) as neo4j_container:\n with neo4j_container.get_driver() as neo4j_driver:\n SparkTest.spark = (\n SparkSession.builder.appName(\"Neo4jConnectorTests\")\n .master(\"local[*]\")\n .config(\"spark.jars\", connector_jar)\n .config(\"spark.driver.host\", \"127.0.0.1\")\n .getOrCreate()\n )\n SparkTest.neo4j_driver = neo4j_driver\n SparkTest.neo4j_container = neo4j_container\n unittest.main()\n SparkTest.spark.close()\n" }, { "path": "scripts/release/upload_to_spark_packages.sh", "content": "#!/bin/bash\n\nset -eEuo pipefail\n\nif [[ $# -lt 5 ]] ; then\n echo \"Usage ./upload_to_spark_packages.sh \"\n exit 1\nfi\n\nUSER=$1\nTOKEN=$2\nGIT_HASH=$3\nVERSION=$4\nPATH_TO_PACKAGE_FILE=$5\n\n# License codes expected:\n# 0 - Apache 2.0\n# 1 - BSD 3-Clause\n# 2 - BSD 2-Clause\n# 3 - GPL-2.0\n# 4 - GPL-3.0\n# 5 - LGPL-2.1\n# 6 - LGPL-3.0\n# 7 - MIT\n# 8 - MPL-2.0\n# 9 - EPL-1.0\n# 10 - Other license\nLICENSE=\"0\"\n\ncurl -X POST 'https://spark-packages.org/api/submit-release' \\\n -u \"$USER:$TOKEN\" \\\n -F \"git_commit_sha1=$GIT_HASH\" \\\n -F \"version=$VERSION\" \\\n -F \"license_id=$LICENSE\" \\\n -F \"name=neo4j/neo4j-spark-connector\" \\\n -F \"artifact_zip=@$PATH_TO_PACKAGE_FILE;type=application/zip\"\n" }, { "path": "spark-3/LICENSES.txt", "content": "This file contains the full license text of the included third party\nlibraries. For an overview of the licenses see the NOTICE.txt file.\n\n\n------------------------------------------------------------------------------\nApache Software License, Version 2.0\n JetBrains Java Annotations\n Kotlin Stdlib\n Netty/Buffer\n Netty/Codec\n Netty/Common\n Netty/Handler\n Netty/Resolver\n Netty/TomcatNative [OpenSSL - Classes]\n Netty/Transport\n Netty/Transport/Native/Unix/Common\n Non-Blocking Reactive Foundation for the JVM\n org.apiguardian:apiguardian-api\n------------------------------------------------------------------------------\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n\n\n------------------------------------------------------------------------------\nMIT License\n SLF4J API Module\n------------------------------------------------------------------------------\n\nThe MIT License\n\nCopyright (c) \n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n\n\n\n------------------------------------------------------------------------------\nMIT No Attribution License\n reactive-streams\n------------------------------------------------------------------------------\n\nMIT No Attribution\n\nCopyright \n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this\nsoftware and associated documentation files (the \"Software\"), to deal in the Software\nwithout restriction, including without limitation the rights to use, copy, modify,\nmerge, publish, distribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,\nINCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\nPARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT\nHOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\n\n\n\n" }, { "path": "spark-3/NOTICE.txt", "content": "Copyright (c) \"Neo4j\"\nNeo4j Sweden AB [https://neo4j.com]\n\nThis file is part of Neo4j.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\nFull license texts are found in LICENSES.txt.\n\n\nThird-party licenses\n--------------------\n\nApache Software License, Version 2.0\n JetBrains Java Annotations\n Kotlin Stdlib\n Netty/Buffer\n Netty/Codec\n Netty/Common\n Netty/Handler\n Netty/Resolver\n Netty/TomcatNative [OpenSSL - Classes]\n Netty/Transport\n Netty/Transport/Native/Unix/Common\n Non-Blocking Reactive Foundation for the JVM\n org.apiguardian:apiguardian-api\n\nMIT License\n SLF4J API Module\n\nMIT No Attribution License\n reactive-streams\n\n" }, { "path": "spark-3/pom.xml", "content": "\n\n 4.0.0\n \n org.neo4j\n neo4j-connector-apache-spark_parent\n 5.4.3-SNAPSHOT\n \n neo4j-connector-apache-spark\n jar\n neo4j-connector-apache-spark-${spark.version}\n Spark ${spark.version} for Neo4j Connector for Apache Spark using the binary Bolt Driver\n \n neo4j-spark-connector\n neo4j\n ${spark-packages.artifactId}-${spark-packages.version}\n \n \n \n \n org.neo4j\n neo4j-connector-apache-spark_common\n ${project.version}\n \n \n org.neo4j.driver\n neo4j-java-driver-slim\n \n \n org.apache.spark\n spark-core_${scala.binary.version}\n provided\n \n \n org.apache.spark\n spark-sql_${scala.binary.version}\n provided\n \n \n org.scala-lang\n scala-library\n provided\n \n \n org.scala-lang\n scala-reflect\n provided\n \n \n org.neo4j\n neo4j-connector-apache-spark_test-support\n ${project.version}\n test\n \n \n org.neo4j.connectors\n commons-authn-keycloak\n test\n \n \n org.scalatest\n scalatest_${scala.binary.version}\n test\n \n \n pl.pragmatists\n JUnitParams\n 1.1.1\n test\n \n \n \n \n \n true\n src/main/resources\n \n \n \n \n net.alchim31.maven\n scala-maven-plugin\n \n \n org.apache.maven.plugins\n maven-failsafe-plugin\n \n \n org.apache.maven.plugins\n maven-surefire-plugin\n \n \n org.apache.maven.plugins\n maven-resources-plugin\n ${maven-resources-plugin.version}\n \n ISO-8859-1\n ${project.build.sourceEncoding}\n \n \n \n maven-assembly-plugin\n 3.8.0\n \n \n bin\n \n single\n \n package\n \n \n src/main/assemblies/spark-packages-assembly.xml\n \n false\n ${spark-packages.packageName}\n \n \n \n \n \n \n\n" }, { "path": "spark-3/src/jreleaser/assemblers/zip/README.txt.tpl", "content": "Neo4j Connector for Apache Spark {{projectVersion}}\n\nThis archive contains release materials for the Neo4j Connector for Apache Spark.\n\n**Make sure you use the correct JAR, depending on the Scala version in use in your\nspark environment**\n\nSource Code & Release Notes: https://github.com/neo4j/neo4j-spark-connector/releases\nDocumentation: https://neo4j.com/docs/spark/current/\n\nNeed Support? Have a Question?\n\nIf you are a Neo4j Enterprise customer, please have a look at the enterprise\nknowledge base, or contact Neo4j Professional Support through:\nhttps://support.neo4j.com/hc/en-us\n\nOtherwise, please consider posting a question on the Neo4j Community site:\nhttps://community.neo4j.com/" }, { "path": "spark-3/src/main/assemblies/spark-packages-assembly.xml", "content": "\n bin\n\n \n zip\n \n\n false\n\n \n \n ${project.build.directory}/${project.artifactId}-${project.version}.jar\n /\n ${spark-packages.packageName}.jar\n \n \n true\n src/main/distributions/spark-packages.pom\n /\n ${spark-packages.packageName}.pom\n \n \n\n\n" }, { "path": "spark-3/src/main/distributions/spark-packages.pom", "content": "\n 4.0.0\n ${spark-packages.groupId}\n ${spark-packages.artifactId}\n ${spark-packages.version}\n\n" }, { "path": "spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister", "content": "org.neo4j.spark.DataSource" }, { "path": "spark-3/src/main/resources/neo4j-spark-connector.properties", "content": "version=${project.version}" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/DataSource.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.sql.SparkSession\nimport org.apache.spark.sql.connector.catalog.Table\nimport org.apache.spark.sql.connector.catalog.TableProvider\nimport org.apache.spark.sql.connector.expressions.Transform\nimport org.apache.spark.sql.sources.DataSourceRegister\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.types.StructType\nimport org.apache.spark.sql.util.CaseInsensitiveStringMap\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.caniuse.Neo4jDetector\nimport org.neo4j.spark.util.DriverCache\nimport org.neo4j.spark.util.Neo4jDriverOptions\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.Neo4jUtil\nimport org.neo4j.spark.util.ValidateConnection\nimport org.neo4j.spark.util.ValidateSparkMinVersion\nimport org.neo4j.spark.util.Validations\n\nimport java.util.UUID\n\nclass DataSource extends TableProvider\n with DataSourceRegister {\n\n Validations.validate(ValidateSparkMinVersion(\"3.3.0\"))\n\n private val jobId: String = UUID.randomUUID().toString\n\n private var schema: StructType = _\n\n private var neo4jOptions: Neo4jOptions = _\n\n private var neo4j: Neo4j = _\n\n override def supportsExternalMetadata(): Boolean = true\n\n override def inferSchema(caseInsensitiveStringMap: CaseInsensitiveStringMap): StructType = {\n if (schema == null) {\n val neo4jOpts = getNeo4jOptions(caseInsensitiveStringMap)\n Validations.validate(ValidateConnection(neo4jOpts, jobId))\n val neo4j = getNeo4jInfo(neo4jOpts.connection)\n schema =\n Neo4jUtil.callSchemaService(\n neo4j,\n neo4jOpts,\n jobId,\n Array.empty[Filter],\n { schemaService =>\n schemaService.struct()\n }\n )\n }\n\n schema\n }\n\n private def getNeo4jInfo(options: Neo4jDriverOptions): Neo4j = {\n if (neo4j == null) {\n val driver = new DriverCache(options).getOrCreate()\n neo4j = Neo4jDetector.INSTANCE.detect(driver)\n }\n neo4j\n }\n\n private def getNeo4jOptions(caseInsensitiveStringMap: CaseInsensitiveStringMap) = {\n if (neo4jOptions == null) {\n neo4jOptions =\n Neo4jOptions.fromSession(SparkSession.getActiveSession, caseInsensitiveStringMap.asCaseSensitiveMap())\n }\n\n neo4jOptions\n }\n\n override def getTable(\n structType: StructType,\n transforms: Array[Transform],\n map: java.util.Map[String, String]\n ): Table = {\n val caseInsensitiveStringMapNeo4jOptions = new CaseInsensitiveStringMap(map)\n val schema = if (structType != null) {\n structType\n } else {\n inferSchema(caseInsensitiveStringMapNeo4jOptions)\n }\n val neo4jOpts = getNeo4jOptions(caseInsensitiveStringMapNeo4jOptions)\n val neo4jInfo = getNeo4jInfo(neo4jOpts.connection)\n new Neo4jTable(neo4jInfo, schema, neo4jOpts, jobId)\n }\n\n override def shortName(): String = \"neo4j\"\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/Neo4jTable.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.internal.Logging\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.connector.catalog.SupportsRead\nimport org.apache.spark.sql.connector.catalog.SupportsWrite\nimport org.apache.spark.sql.connector.catalog.Table\nimport org.apache.spark.sql.connector.catalog.TableCapability\nimport org.apache.spark.sql.connector.write.LogicalWriteInfo\nimport org.apache.spark.sql.connector.write.WriteBuilder\nimport org.apache.spark.sql.types.StructType\nimport org.apache.spark.sql.util.CaseInsensitiveStringMap\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.driver.AccessMode\nimport org.neo4j.spark.reader.Neo4jScanBuilder\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.ValidateRead\nimport org.neo4j.spark.util.Validations\nimport org.neo4j.spark.writer.Neo4jWriterBuilder\n\nimport scala.collection.JavaConverters._\n\nclass Neo4jTable(neo4j: Neo4j, schema: StructType, neo4jOptions: Neo4jOptions, jobId: String)\n extends Table\n with SupportsRead\n with SupportsWrite\n with Logging {\n\n override def name(): String = neo4jOptions.getTableName\n\n override def schema(): StructType = schema\n\n override def capabilities(): java.util.Set[TableCapability] = Set(\n TableCapability.BATCH_READ,\n TableCapability.BATCH_WRITE,\n TableCapability.ACCEPT_ANY_SCHEMA,\n TableCapability.OVERWRITE_BY_FILTER,\n TableCapability.OVERWRITE_DYNAMIC,\n TableCapability.STREAMING_WRITE,\n TableCapability.MICRO_BATCH_READ\n ).asJava\n\n override def newScanBuilder(options: CaseInsensitiveStringMap): Neo4jScanBuilder = {\n Validations.validate(ValidateRead(neo4j, neo4jOptions, jobId))\n new Neo4jScanBuilder(neo4j, neo4jOptions, jobId, schema())\n }\n\n override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {\n val mapOptions = neo4jOptions.asMap()\n mapOptions.put(Neo4jOptions.ACCESS_MODE, AccessMode.WRITE.toString)\n val writeNeo4jOptions = new Neo4jOptions(mapOptions)\n new Neo4jWriterBuilder(neo4j, info.queryId(), info.schema(), SaveMode.Append, writeNeo4jOptions)\n }\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/reader/Neo4jPartitionReader.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.reader\n\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.connector.expressions.aggregate.AggregateFunc\nimport org.apache.spark.sql.connector.read.PartitionReader\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.service.PartitionPagination\nimport org.neo4j.spark.util.Neo4jOptions\n\nclass Neo4jPartitionReader(\n private val neo4j: Neo4j,\n private val options: Neo4jOptions,\n private val filters: Array[Filter],\n private val schema: StructType,\n private val jobId: String,\n private val partitionSkipLimit: PartitionPagination,\n private val scriptResult: java.util.List[java.util.Map[String, AnyRef]],\n private val requiredColumns: StructType,\n private val aggregateColumns: Array[AggregateFunc]\n) extends BasePartitionReader(\n neo4j,\n options,\n filters,\n schema,\n jobId,\n partitionSkipLimit,\n scriptResult,\n requiredColumns,\n aggregateColumns\n )\n with PartitionReader[InternalRow]\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/reader/Neo4jPartitionReaderFactory.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.reader\n\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.connector.expressions.aggregate.AggregateFunc\nimport org.apache.spark.sql.connector.read.InputPartition\nimport org.apache.spark.sql.connector.read.PartitionReader\nimport org.apache.spark.sql.connector.read.PartitionReaderFactory\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.service.PartitionPagination\nimport org.neo4j.spark.util.Neo4jOptions\n\nclass Neo4jPartitionReaderFactory(\n private val neo4j: Neo4j,\n private val neo4jOptions: Neo4jOptions,\n private val filters: Array[Filter],\n private val schema: StructType,\n private val jobId: String,\n private val scriptResult: java.util.List[java.util.Map[String, AnyRef]],\n private val requiredColumns: StructType,\n private val aggregateColumns: Array[AggregateFunc]\n) extends PartitionReaderFactory {\n\n override def createReader(partition: InputPartition): PartitionReader[InternalRow] = new Neo4jPartitionReader(\n neo4j,\n neo4jOptions,\n filters,\n schema,\n jobId,\n partition.asInstanceOf[Neo4jPartition].partitionSkipLimit,\n scriptResult,\n requiredColumns,\n aggregateColumns\n )\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/reader/Neo4jScan.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.reader\n\nimport org.apache.spark.sql.connector.expressions.aggregate.AggregateFunc\nimport org.apache.spark.sql.connector.read.Batch\nimport org.apache.spark.sql.connector.read.InputPartition\nimport org.apache.spark.sql.connector.read.PartitionReaderFactory\nimport org.apache.spark.sql.connector.read.Scan\nimport org.apache.spark.sql.connector.read.streaming.MicroBatchStream\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.config.TopN\nimport org.neo4j.spark.service.PartitionPagination\nimport org.neo4j.spark.streaming.Neo4jMicroBatchReader\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.Neo4jUtil\nimport org.neo4j.spark.util.ValidateReadNotStreaming\nimport org.neo4j.spark.util.ValidateReadStreaming\nimport org.neo4j.spark.util.Validations\n\ncase class Neo4jPartition(partitionSkipLimit: PartitionPagination) extends InputPartition\n\nclass Neo4jScan(\n neo4j: Neo4j,\n neo4jOptions: Neo4jOptions,\n jobId: String,\n schema: StructType,\n filters: Array[Filter],\n requiredColumns: StructType,\n aggregateColumns: Array[AggregateFunc],\n topN: Option[TopN]\n) extends Scan with Batch {\n\n override def toBatch: Batch = this\n\n var scriptResult: java.util.List[java.util.Map[String, AnyRef]] = _\n\n private def createPartitions() = {\n Validations.validate(ValidateReadNotStreaming(neo4jOptions, jobId))\n // we get the skip/limit for each partition and execute the \"script\"\n val (partitionSkipLimitList, scriptResult) = Neo4jUtil.callSchemaService(\n neo4j,\n neo4jOptions,\n jobId,\n filters,\n { schemaService =>\n (schemaService.skipLimitFromPartition(topN), schemaService.execute(neo4jOptions.script))\n }\n )\n // we generate a partition for each element\n this.scriptResult = scriptResult\n partitionSkipLimitList\n .map(partitionSkipLimit => Neo4jPartition(partitionSkipLimit))\n }\n\n override def planInputPartitions(): Array[InputPartition] = {\n val neo4jPartitions: Seq[Neo4jPartition] = createPartitions()\n neo4jPartitions.toArray\n }\n\n override def createReaderFactory(): PartitionReaderFactory = {\n new Neo4jPartitionReaderFactory(\n neo4j,\n neo4jOptions,\n filters,\n schema,\n jobId,\n scriptResult,\n requiredColumns,\n aggregateColumns\n )\n }\n\n override def readSchema(): StructType = schema\n\n override def toMicroBatchStream(checkpointLocation: String): MicroBatchStream = {\n Validations.validate(ValidateReadStreaming(neo4j, neo4jOptions, jobId))\n new Neo4jMicroBatchReader(neo4j, schema, neo4jOptions, jobId, aggregateColumns)\n }\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/reader/Neo4jScanBuilder.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.reader\n\nimport org.apache.spark.internal.Logging\nimport org.apache.spark.sql.connector.expressions.Expression\nimport org.apache.spark.sql.connector.expressions.SortOrder\nimport org.apache.spark.sql.connector.expressions.aggregate.AggregateFunc\nimport org.apache.spark.sql.connector.expressions.aggregate.Aggregation\nimport org.apache.spark.sql.connector.expressions.filter.Predicate\nimport org.apache.spark.sql.connector.read.Scan\nimport org.apache.spark.sql.connector.read.SupportsPushDownAggregates\nimport org.apache.spark.sql.connector.read.SupportsPushDownLimit\nimport org.apache.spark.sql.connector.read.SupportsPushDownRequiredColumns\nimport org.apache.spark.sql.connector.read.SupportsPushDownTopN\nimport org.apache.spark.sql.connector.read.SupportsPushDownV2Filters\nimport org.apache.spark.sql.types.LongType\nimport org.apache.spark.sql.types.StructField\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.config.TopN\nimport org.neo4j.spark.util.Neo4jImplicits.AggregationImplicit\nimport org.neo4j.spark.util.Neo4jImplicits.CypherImplicits\nimport org.neo4j.spark.util.Neo4jImplicits.PredicateImplicit\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.QueryType\n\nclass Neo4jScanBuilder(neo4jInfo: Neo4j, neo4jOptions: Neo4jOptions, jobId: String, schema: StructType)\n extends SupportsPushDownV2Filters\n with SupportsPushDownAggregates\n with SupportsPushDownRequiredColumns\n with SupportsPushDownLimit\n with SupportsPushDownTopN\n with Logging {\n\n private var predicates: Array[Predicate] = Array.empty\n\n private var requiredSchema: StructType = schema\n private var requiredColumns: StructType = new StructType()\n\n private var aggregateColumns: Array[AggregateFunc] = Array.empty[AggregateFunc]\n private var limit: Option[Int] = None\n private var topN: Option[TopN] = None\n\n override def build(): Scan = {\n new Neo4jScan(\n neo4jInfo,\n neo4jOptions,\n jobId,\n requiredSchema,\n predicates.flatMap(_.toFilter(neo4jOptions)),\n requiredColumns,\n aggregateColumns,\n topN.orElse(limit.map((limit: Int) => TopN(limit)))\n )\n }\n\n override def pushPredicates(predicatesArray: Array[Predicate]): Array[Predicate] = {\n if (neo4jOptions.pushdownFiltersEnabled) {\n predicates = predicatesArray\n }\n\n predicatesArray\n }\n\n override def pushedPredicates(): Array[Predicate] = predicates\n\n override def pruneColumns(newSchema: StructType): Unit = {\n if (!neo4jOptions.pushdownColumnsEnabled || neo4jOptions.relationshipMetadata.nodeMap) {\n new StructType()\n } else {\n requiredColumns = StructType(requiredSchema.filter(sf => newSchema.contains(sf)))\n }\n }\n\n override def pushAggregation(aggregation: Aggregation): Boolean = {\n if (\n !neo4jOptions.pushdownAggregateEnabled\n || aggregation.aggregateExpressions().isEmpty\n || neo4jOptions.query.queryType == QueryType.QUERY\n ) {\n return false\n }\n aggregateColumns = aggregation.aggregateExpressions()\n val groupByColumns: Set[String] = aggregation.groupByCols()\n .map(_.describe().unquote())\n .toSet\n requiredColumns = StructType(requiredSchema.filter(field => groupByColumns.contains(field.name)))\n\n aggregateColumns.foreach(af => {\n val fields =\n try {\n af.children()\n .toSet[Expression]\n .map(_.describe())\n .map(_.unquote())\n } catch {\n // for making it compatible with Spark 3.2\n case noSuchMethodException: NoSuchMethodError => Set(af.describe().unquote())\n }\n val dt = if (fields.nonEmpty) {\n requiredSchema.filter(field => fields.contains(field.name))\n .map(_.dataType)\n .toSet\n .headOption\n .getOrElse(LongType)\n } else {\n LongType\n }\n requiredColumns = requiredColumns.add(StructField(af.toString, dt))\n })\n requiredSchema = requiredColumns\n true\n }\n\n override def pushLimit(pushedLimit: Int): Boolean = {\n if (!neo4jOptions.pushdownLimitEnabled) {\n return false\n }\n if (neo4jOptions.partitions > 1) {\n logWarning(\n s\"\"\"Disabling pushed down limit support since it conflicts with partitioning.\n |Set the `${Neo4jOptions.PARTITIONS}` parameter value to 1\n | or set `${Neo4jOptions.PUSHDOWN_LIMIT_ENABLED}` to false to remove this warning.\n |\"\"\".stripMargin\n )\n return false\n }\n if (pushedLimit <= 0) {\n logWarning(s\"Ignoring negative pushed down limit $pushedLimit.\")\n return false\n }\n limit = Some(pushedLimit)\n true\n }\n\n override def pushTopN(orders: Array[SortOrder], limit: Int): Boolean = {\n if (!neo4jOptions.pushdownTopNEnabled) {\n return false\n }\n if (limit > 0 && neo4jOptions.partitions > 1) {\n logWarning(\"disabling pushed down top N support since it conflicts with partitioning.\" +\n \"set the partition count to 1 or disable the pushdown limit support to remove this warning\")\n return false\n }\n topN = Some(TopN(limit, orders))\n true\n }\n\n // otherwise doesn't compile in Scala 2.12\n override def isPartiallyPushed: Boolean = true\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/streaming/Neo4jMicroBatchReader.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.streaming\n\nimport org.apache.spark.internal.Logging\nimport org.apache.spark.sql.connector.expressions.aggregate.AggregateFunc\nimport org.apache.spark.sql.connector.read.InputPartition\nimport org.apache.spark.sql.connector.read.PartitionReaderFactory\nimport org.apache.spark.sql.connector.read.streaming.MicroBatchStream\nimport org.apache.spark.sql.connector.read.streaming.Offset\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.sources.GreaterThan\nimport org.apache.spark.sql.sources.LessThanOrEqual\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.service.SchemaService\nimport org.neo4j.spark.util._\n\nclass Neo4jMicroBatchReader(\n private val neo4j: Neo4j,\n private val schema: StructType,\n private val neo4jOptions: Neo4jOptions,\n private val jobId: String,\n private val aggregateColumns: Array[AggregateFunc]\n) extends MicroBatchStream\n with Logging {\n\n private val driverCache = new DriverCache(neo4jOptions.connection)\n\n private lazy val scriptResult = {\n val schemaService = new SchemaService(neo4j, neo4jOptions, driverCache)\n schemaService.createOptimizations(schema)\n val scriptResult = schemaService.execute(neo4jOptions.script)\n schemaService.close()\n scriptResult\n }\n\n private var filters: Array[Filter] = Array.empty[Filter]\n\n override def deserializeOffset(json: String): Offset = Neo4jOffset(json.toLong)\n\n override def commit(end: Offset): Unit = {}\n\n override def planInputPartitions(start: Offset, end: Offset): Array[InputPartition] = {\n logDebug(s\"start and end offset: $start - $end\")\n\n val prop = Neo4jUtil.getStreamingPropertyName(neo4jOptions)\n this.filters =\n Array(\n GreaterThan(prop, start.asInstanceOf[Neo4jOffset].offset),\n LessThanOrEqual(prop, end.asInstanceOf[Neo4jOffset].offset)\n )\n\n val partitions = Neo4jUtil.callSchemaService(\n neo4j,\n neo4jOptions,\n jobId,\n filters,\n { schemaService => schemaService.skipLimitFromPartition(None) }\n )\n\n partitions\n .map(p => Neo4jStreamingPartition(p, filters))\n .toArray\n }\n\n override def stop(): Unit = {\n driverCache.close()\n }\n\n override def latestOffset(): Offset = {\n val offsetValue = Neo4jUtil.callSchemaService[Option[Long]](\n neo4j,\n neo4jOptions,\n jobId,\n filters,\n {\n schemaService =>\n try {\n schemaService.lastOffset()\n } catch {\n case _: Throwable => null\n }\n }\n )\n offsetValue.map(value => Neo4jOffset(value)).orNull\n }\n\n override def initialOffset(): Offset = Neo4jOffset(neo4jOptions.streamingOptions.from.value())\n\n override def createReaderFactory(): PartitionReaderFactory = {\n new Neo4jStreamingPartitionReaderFactory(\n neo4j,\n neo4jOptions,\n schema,\n jobId,\n scriptResult,\n aggregateColumns\n )\n }\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/streaming/Neo4jOffset.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.streaming\n\nimport org.apache.spark.sql.connector.read.streaming.Offset\n\ncase class Neo4jOffset(offset: Long) extends Offset {\n override def json(): String = offset.toString\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/streaming/Neo4jStreamingDataWriterFactory.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.streaming\n\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.connector.write.DataWriter\nimport org.apache.spark.sql.connector.write.streaming.StreamingDataWriterFactory\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.writer.Neo4jDataWriter\n\nclass Neo4jStreamingDataWriterFactory(\n neo4j: Neo4j,\n jobId: String,\n schema: StructType,\n saveMode: SaveMode,\n options: Neo4jOptions,\n scriptResult: java.util.List[java.util.Map[String, AnyRef]]\n) extends StreamingDataWriterFactory {\n\n override def createWriter(partitionId: Int, taskId: Long, epochId: Long): DataWriter[InternalRow] =\n new Neo4jDataWriter(\n neo4j,\n jobId,\n partitionId,\n schema,\n saveMode,\n options,\n scriptResult\n )\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/streaming/Neo4jStreamingPartitionReader.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.streaming\n\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.connector.expressions.aggregate.AggregateFunc\nimport org.apache.spark.sql.connector.read.PartitionReader\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.types.StructType\nimport org.apache.spark.util.AccumulatorV2\nimport org.apache.spark.util.LongAccumulator\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.service.PartitionPagination\nimport org.neo4j.spark.util.Neo4jOptions\n\nclass Neo4jStreamingPartitionReader(\n private val neo4j: Neo4j,\n private val options: Neo4jOptions,\n private val filters: Array[Filter],\n private val schema: StructType,\n private val jobId: String,\n private val partitionSkipLimit: PartitionPagination,\n private val scriptResult: java.util.List[java.util.Map[String, AnyRef]],\n private val requiredColumns: StructType,\n private val aggregateColumns: Array[AggregateFunc]\n) extends BaseStreamingPartitionReader(\n neo4j,\n options,\n filters,\n schema,\n jobId,\n partitionSkipLimit,\n scriptResult,\n requiredColumns,\n aggregateColumns\n )\n with PartitionReader[InternalRow] {}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/streaming/Neo4jStreamingPartitionReaderFactory.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.streaming\n\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.connector.expressions.aggregate.AggregateFunc\nimport org.apache.spark.sql.connector.read.InputPartition\nimport org.apache.spark.sql.connector.read.PartitionReader\nimport org.apache.spark.sql.connector.read.PartitionReaderFactory\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.types.StructType\nimport org.apache.spark.util.AccumulatorV2\nimport org.apache.spark.util.LongAccumulator\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.service.PartitionPagination\nimport org.neo4j.spark.util.Neo4jOptions\n\ncase class Neo4jStreamingPartition(partitionSkipLimit: PartitionPagination, filters: Array[Filter])\n extends InputPartition\n\nclass Neo4jStreamingPartitionReaderFactory(\n private val neo4j: Neo4j,\n private val neo4jOptions: Neo4jOptions,\n private val schema: StructType,\n private val jobId: String,\n private val scriptResult: java.util.List[java.util.Map[String, AnyRef]],\n private val aggregateColumns: Array[AggregateFunc]\n) extends PartitionReaderFactory {\n\n override def createReader(partition: InputPartition): PartitionReader[InternalRow] =\n new Neo4jStreamingPartitionReader(\n neo4j,\n neo4jOptions,\n partition.asInstanceOf[Neo4jStreamingPartition].filters,\n schema,\n jobId,\n partition.asInstanceOf[Neo4jStreamingPartition].partitionSkipLimit,\n scriptResult,\n new StructType(),\n aggregateColumns\n )\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/streaming/Neo4jStreamingWriter.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.streaming\n\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.SparkSession\nimport org.apache.spark.sql.connector.write.PhysicalWriteInfo\nimport org.apache.spark.sql.connector.write.WriterCommitMessage\nimport org.apache.spark.sql.connector.write.streaming.StreamingDataWriterFactory\nimport org.apache.spark.sql.connector.write.streaming.StreamingWrite\nimport org.apache.spark.sql.streaming.StreamingQueryListener\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.service.SchemaService\nimport org.neo4j.spark.util.DriverCache\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.Neo4jUtil\n\nimport java.util.Optional\n\nclass Neo4jStreamingWriter(\n val neo4j: Neo4j,\n val queryId: String,\n val schema: StructType,\n saveMode: SaveMode,\n val neo4jOptions: Neo4jOptions\n) extends StreamingWrite {\n\n private val self = this\n\n private val listener = new StreamingQueryListener {\n override def onQueryStarted(event: StreamingQueryListener.QueryStartedEvent): Unit = ()\n\n override def onQueryProgress(event: StreamingQueryListener.QueryProgressEvent): Unit = ()\n\n override def onQueryTerminated(event: StreamingQueryListener.QueryTerminatedEvent): Unit = {\n if (event.id.toString == queryId) {\n self.close()\n SparkSession.getDefaultSession.get.streams.removeListener(this)\n }\n }\n }\n SparkSession.getDefaultSession.get.streams.addListener(listener)\n\n private val driverCache = new DriverCache(neo4jOptions.connection)\n\n private lazy val scriptResult = {\n val schemaService = new SchemaService(neo4j, neo4jOptions, driverCache)\n schemaService.createOptimizations(schema)\n val scriptResult = schemaService.execute(neo4jOptions.script)\n schemaService.close()\n scriptResult\n }\n\n override def createStreamingWriterFactory(info: PhysicalWriteInfo): StreamingDataWriterFactory = {\n new Neo4jStreamingDataWriterFactory(\n neo4j,\n queryId,\n schema,\n saveMode,\n neo4jOptions,\n scriptResult\n )\n }\n\n override def commit(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}\n\n override def abort(epochId: Long, messages: Array[WriterCommitMessage]): Unit = {}\n\n def close(): Unit = Neo4jUtil.closeSafely(driverCache)\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/writer/Neo4jBatchWriter.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.writer\n\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.connector.write.BatchWrite\nimport org.apache.spark.sql.connector.write.DataWriterFactory\nimport org.apache.spark.sql.connector.write.PhysicalWriteInfo\nimport org.apache.spark.sql.connector.write.WriterCommitMessage\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.service.SchemaService\nimport org.neo4j.spark.util.DriverCache\nimport org.neo4j.spark.util.Neo4jOptions\n\nimport java.util.Optional\n\nclass Neo4jBatchWriter(\n neo4j: Neo4j,\n jobId: String,\n structType: StructType,\n saveMode: SaveMode,\n neo4jOptions: Neo4jOptions\n) extends BatchWrite {\n\n override def createBatchWriterFactory(physicalWriteInfo: PhysicalWriteInfo): DataWriterFactory = {\n val schemaService = new SchemaService(neo4j, neo4jOptions, driverCache)\n schemaService.createOptimizations(structType)\n val scriptResult = schemaService.execute(neo4jOptions.script)\n schemaService.close()\n\n if (neo4jOptions.indexAwait > 0) {\n val session = driverCache.getOrCreate().session(neo4jOptions.session.toNeo4jSession())\n session.run(s\"CALL db.awaitIndexes(${neo4jOptions.indexAwait})\").consume()\n }\n\n new Neo4jDataWriterFactory(\n neo4j,\n jobId,\n structType,\n saveMode,\n neo4jOptions,\n scriptResult\n )\n }\n\n private val driverCache = new DriverCache(neo4jOptions.connection)\n\n override def commit(messages: Array[WriterCommitMessage]): Unit = {\n driverCache.close()\n }\n\n override def abort(messages: Array[WriterCommitMessage]): Unit = {\n driverCache.close()\n }\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/writer/Neo4jDataWriter.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.writer\n\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.connector.write.DataWriter\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.util.Neo4jOptions\n\nclass Neo4jDataWriter(\n neo4j: Neo4j,\n jobId: String,\n partitionId: Int,\n schema: StructType,\n saveMode: SaveMode,\n options: Neo4jOptions,\n scriptResult: java.util.List[java.util.Map[String, AnyRef]]\n) extends BaseDataWriter(neo4j, jobId, partitionId, schema, saveMode, options, scriptResult)\n with DataWriter[InternalRow] {}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/writer/Neo4jDataWriterFactory.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.writer\n\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.catalyst.InternalRow\nimport org.apache.spark.sql.connector.write.DataWriter\nimport org.apache.spark.sql.connector.write.DataWriterFactory\nimport org.apache.spark.sql.connector.write.PhysicalWriteInfo\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.util.Neo4jOptions\n\nclass Neo4jDataWriterFactory(\n neo4j: Neo4j,\n jobId: String,\n structType: StructType,\n saveMode: SaveMode,\n options: Neo4jOptions,\n scriptResult: java.util.List[java.util.Map[String, AnyRef]]\n) extends DataWriterFactory {\n\n override def createWriter(partitionId: Int, taskId: Long): DataWriter[InternalRow] = new Neo4jDataWriter(\n neo4j,\n jobId,\n partitionId,\n structType,\n saveMode,\n options,\n scriptResult\n )\n}\n" }, { "path": "spark-3/src/main/scala/org/neo4j/spark/writer/Neo4jWriterBuilder.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark.writer\n\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.connector.metric.CustomMetric\nimport org.apache.spark.sql.connector.write.BatchWrite\nimport org.apache.spark.sql.connector.write.SupportsOverwrite\nimport org.apache.spark.sql.connector.write.SupportsTruncate\nimport org.apache.spark.sql.connector.write.Write\nimport org.apache.spark.sql.connector.write.WriteBuilder\nimport org.apache.spark.sql.connector.write.streaming.StreamingWrite\nimport org.apache.spark.sql.sources.Filter\nimport org.apache.spark.sql.types.StructType\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.spark.streaming.Neo4jStreamingWriter\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.NodeSaveMode\nimport org.neo4j.spark.util.ValidateSaveMode\nimport org.neo4j.spark.util.ValidateWrite\nimport org.neo4j.spark.util.ValidationUtil\nimport org.neo4j.spark.util.Validations\n\nclass Neo4jWriterBuilder(\n neo4j: Neo4j,\n queryId: String,\n schema: StructType,\n saveMode: SaveMode,\n neo4jOptions: Neo4jOptions\n) extends WriteBuilder\n with SupportsOverwrite\n with SupportsTruncate {\n\n override def build(): Write = new Write {\n override def description(): String = \"Neo4j Writer\"\n\n override def toBatch: BatchWrite = buildForBatch()\n\n override def toStreaming: StreamingWrite = buildForStreaming()\n\n override def supportedCustomMetrics(): Array[CustomMetric] = DataWriterMetrics.metricDeclarations()\n }\n\n def validOptions(actualSaveMode: SaveMode): Neo4jOptions = {\n Validations.validate(ValidateWrite(\n neo4j,\n neo4jOptions,\n queryId,\n actualSaveMode,\n (o: Neo4jOptions) => {\n ValidationUtil.isFalse(\n o.relationshipMetadata.sourceSaveMode.equals(NodeSaveMode.ErrorIfExists)\n && o.relationshipMetadata.targetSaveMode.equals(NodeSaveMode.ErrorIfExists),\n \"Save mode 'ErrorIfExists' is not supported on Spark 3.0, use 'Append' instead.\"\n )\n }\n ))\n neo4jOptions\n }\n\n override def buildForBatch(): BatchWrite =\n new Neo4jBatchWriter(neo4j, queryId, schema, saveMode, validOptions(saveMode))\n\n @volatile\n private var streamWriter: Neo4jStreamingWriter = _\n\n def isNewInstance(queryId: String, schema: StructType, options: Neo4jOptions): Boolean =\n streamWriter == null ||\n streamWriter.queryId != queryId ||\n streamWriter.schema != schema ||\n streamWriter.neo4jOptions != options\n\n override def buildForStreaming(): StreamingWrite = {\n if (isNewInstance(queryId, schema, neo4jOptions)) {\n val streamingSaveMode = neo4jOptions.saveMode\n Validations.validate(ValidateSaveMode(streamingSaveMode))\n\n val saveMode = SaveMode.valueOf(streamingSaveMode)\n streamWriter = new Neo4jStreamingWriter(\n neo4j,\n queryId,\n schema,\n saveMode,\n validOptions(saveMode)\n )\n }\n\n streamWriter\n }\n\n override def overwrite(filters: Array[Filter]): WriteBuilder = {\n new Neo4jWriterBuilder(neo4j, queryId, schema, SaveMode.Overwrite, neo4jOptions)\n }\n}\n" }, { "path": "spark-3/src/test/java/org/neo4j/spark/DataSourceReaderTypesTSE.java", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark;\n\nimport org.apache.spark.sql.Dataset;\nimport org.apache.spark.sql.Row;\nimport org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema;\nimport org.junit.Test;\nimport org.neo4j.driver.Session;\n\nimport java.sql.Timestamp;\nimport java.time.*;\nimport java.sql.Date;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.TimeZone;\n\nimport static org.junit.Assert.assertEquals;\n\npublic class DataSourceReaderTypesTSE extends SparkConnectorScalaBaseTSE {\n\n @Test\n public void testReadNodeWithString() {\n String name = \"Foobar\";\n Dataset df = initTest(\"CREATE (p:Person {name: '\" + name + \"'})\");\n\n assertEquals(name, df.select(\"name\").collectAsList().get(0).getString(0));\n }\n\n @Test\n public void testReadNodeWithLong() {\n long age = 42L;\n Dataset df = initTest(\"CREATE (p:Person {age: \" + age + \"})\");\n\n assertEquals(age, df.select(\"age\").collectAsList().get(0).getLong(0));\n }\n\n @Test\n public void testReadNodeWithDouble() {\n double score = 4.2;\n Dataset df = initTest(\"CREATE (p:Person {score: \" + score + \"})\");\n\n assertEquals(score, df.select(\"score\").collectAsList().get(0).getDouble(0), 0);\n }\n\n @Test\n public void testReadNodeWithLocalTime() {\n Dataset df = initTest(\"CREATE (p:Person {aTime: localtime({hour:12, minute: 23, second: 0, millisecond: 294})})\");\n\n GenericRowWithSchema result = df.select(\"aTime\").collectAsList().get(0).getAs(0);\n\n assertEquals(\"local-time\", result.get(0));\n assertEquals(\"12:23:00.294\", result.get(1));\n }\n\n @Test\n public void testReadNodeWithTime() {\n TimeZone timezone = TimeZone.getDefault();\n Dataset df = initTest(\"CREATE (p:Person {aTime: time({hour:12, minute: 23, second: 0, millisecond: 294, timezone: '\" + timezone.getID() + \"'})})\");\n\n GenericRowWithSchema result = df.select(\"aTime\").collectAsList().get(0).getAs(0);\n\n LocalTime localTime = LocalTime.of(12, 23, 0, 294000000);\n OffsetTime expectedTime = OffsetTime.of(localTime, timezone.toZoneId().getRules().getOffset(Instant.now()));\n\n assertEquals(\"offset-time\", result.get(0));\n assertEquals(expectedTime.toString(), result.get(1));\n }\n\n @Test\n public void testReadNodeWithLocalDateTime() {\n String localDateTime = \"2007-12-03T10:15:30\";\n Dataset df = initTest(\"CREATE (p:Person {aTime: localdatetime('\" + localDateTime + \"')})\");\n\n Row row = df.select(\"aTime\").collectAsList().get(0);\n LocalDateTime result = row.getAs(0);\n assertEquals(LocalDateTime.parse(localDateTime), result);\n }\n\n @Test\n public void testReadNodeWithZonedDateTime() {\n String dateTime = \"2015-06-24T12:50:35.556+01:00\";\n Dataset df = initTest(\"CREATE (p:Person {aTime: datetime('\" + dateTime + \"')})\");\n\n Timestamp result = df.select(\"aTime\").collectAsList().get(0).getTimestamp(0);\n assertEquals(Timestamp.from(OffsetDateTime.parse(dateTime).toInstant()), result);\n }\n\n @Test\n public void testReadNodeWithPoint() {\n Dataset df = initTest(\"CREATE (p:Person {location: point({x: 12.12, y: 13.13})})\");\n\n GenericRowWithSchema res = df.select(\"location\").collectAsList().get(0).getAs(0);\n assertEquals(\"point-2d\", res.get(0));\n assertEquals(7203, res.get(1));\n assertEquals(12.12, res.get(2));\n assertEquals(13.13, res.get(3));\n }\n\n @Test\n public void testReadNodeWithGeoPoint() {\n Dataset df = initTest(\"CREATE (p:Person {location: point({longitude: 12.12, latitude: 13.13})})\");\n\n GenericRowWithSchema res = df.select(\"location\").collectAsList().get(0).getAs(0);\n assertEquals(\"point-2d\", res.get(0));\n assertEquals(4326, res.get(1));\n assertEquals(12.12, res.get(2));\n assertEquals(13.13, res.get(3));\n }\n\n @Test\n public void testReadNodeWithPoint3D() {\n Dataset df = initTest(\"CREATE (p:Person {location: point({x: 12.12, y: 13.13, z: 1})})\");\n\n GenericRowWithSchema res = df.select(\"location\").collectAsList().get(0).getAs(0);\n assertEquals(\"point-3d\", res.get(0));\n assertEquals(9157, res.get(1));\n assertEquals(12.12, res.get(2));\n assertEquals(13.13, res.get(3));\n assertEquals(1.0, res.get(4));\n }\n\n @Test\n public void testReadNodeWithDate() {\n Dataset df = initTest(\"CREATE (p:Person {born: date('2009-10-10')})\");\n\n Date res = df.select(\"born\").collectAsList().get(0).getDate(0);\n assertEquals(Date.valueOf(\"2009-10-10\"), res);\n }\n\n @Test\n public void testReadNodeWithDuration() {\n Dataset df = initTest(\"CREATE (p:Person {range: duration({days: 14, hours:16, minutes: 12})})\");\n\n GenericRowWithSchema res = df.select(\"range\").collectAsList().get(0).getAs(0);\n assertEquals(\"duration\", res.get(0));\n assertEquals(0L, res.get(1));\n assertEquals(14L, res.get(2));\n assertEquals(58320L, res.get(3));\n assertEquals(0, res.get(4));\n assertEquals(\"P0M14DT58320S\", res.get(5));\n }\n\n @Test\n public void testReadNodeWithStringArray() {\n Dataset df = initTest(\"CREATE (p:Person {names: ['John', 'Doe']})\");\n\n List res = df.select(\"names\").collectAsList().get(0).getList(0);\n assertEquals(\"John\", res.get(0));\n assertEquals(\"Doe\", res.get(1));\n }\n\n @Test\n public void testReadNodeWithLongArray() {\n Dataset df = initTest(\"CREATE (p:Person {ages: [22, 23]})\");\n\n List res = df.select(\"ages\").collectAsList().get(0).getList(0);\n assertEquals(22L, res.get(0).longValue());\n assertEquals(23L, res.get(1).longValue());\n }\n\n @Test\n public void testReadNodeWithDoubleArray() {\n Dataset df = initTest(\"CREATE (p:Person {scores: [22.33, 44.55]})\");\n\n List res = df.select(\"scores\").collectAsList().get(0).getList(0);\n assertEquals(22.33, res.get(0), 0);\n assertEquals(44.55, res.get(1), 0);\n }\n\n @Test\n public void testReadNodeWithLocalTimeArray() {\n Dataset df = initTest(\"CREATE (p:Person {someTimes: [localtime({hour:12}), localtime({hour:1, minute: 3})]})\");\n\n List res = df.select(\"someTimes\").collectAsList().get(0).getList(0);\n assertEquals(\"local-time\", res.get(0).get(0));\n assertEquals(\"12:00:00\", res.get(0).get(1));\n assertEquals(\"local-time\", res.get(1).get(0));\n assertEquals(\"01:03:00\", res.get(1).get(1));\n }\n\n @Test\n public void testReadNodeWithBooleanArray() {\n Dataset df = initTest(\"CREATE (p:Person {bools: [true, false]})\");\n\n List res = df.select(\"bools\").collectAsList().get(0).getList(0);\n assertEquals(true, res.get(0));\n assertEquals(false, res.get(1));\n }\n\n @Test\n public void testReadNodeWithArrayDate() {\n Dataset df = initTest(\"CREATE (p:Person {dates: [date('2009-10-10'), date('2009-10-11')]})\");\n\n List res = df.select(\"dates\").collectAsList().get(0).getList(0);\n assertEquals(Date.valueOf(\"2009-10-10\"), res.get(0));\n assertEquals(Date.valueOf(\"2009-10-11\"), res.get(1));\n }\n\n @Test\n public void testReadNodeWithArrayZonedDateTime() {\n String datetime1 = \"2015-06-24T12:50:35.556+01:00\";\n String datetime2 = \"2015-06-23T12:50:35.556+01:00\";\n Dataset df = initTest(\"CREATE (p:Person {dates: [datetime('\" + datetime1 + \"'), datetime('\" + datetime2 + \"')]})\");\n\n List res = df.select(\"dates\").collectAsList().get(0).getList(0);\n assertEquals(Timestamp.from(OffsetDateTime.parse(datetime1).toInstant()), res.get(0));\n assertEquals(Timestamp.from(OffsetDateTime.parse(datetime2).toInstant()), res.get(1));\n }\n\n @Test\n public void testReadNodeWithArrayDurations() {\n Dataset df = initTest(\"CREATE (p:Person {durations: [duration({months: 0.75}), duration({weeks: 2.5})]})\");\n\n List res = df.select(\"durations\").collectAsList().get(0).getList(0);\n assertEquals(\"duration\", res.get(0).get(0));\n assertEquals(0L, res.get(0).get(1));\n assertEquals(22L, res.get(0).get(2));\n assertEquals(71509L, res.get(0).get(3));\n assertEquals(500000000, res.get(0).get(4));\n assertEquals(\"P0M22DT71509.500000000S\", res.get(0).get(5));\n\n assertEquals(\"duration\", res.get(1).get(0));\n assertEquals(0L, res.get(1).get(1));\n assertEquals(17L, res.get(1).get(2));\n assertEquals(43200L, res.get(1).get(3));\n assertEquals(0, res.get(1).get(4));\n assertEquals(\"P0M17DT43200S\", res.get(1).get(5));\n }\n\n @Test\n public void testReadNodeWithPointArray() {\n Dataset df = initTest(\"CREATE (p:Person {locations: [point({x: 11, y: 33.111}), point({x: 22, y: 44.222})]})\");\n\n List res = df.select(\"locations\").collectAsList().get(0).getList(0);\n assertEquals(\"point-2d\", res.get(0).get(0));\n assertEquals(7203, res.get(0).get(1));\n assertEquals(11.0, res.get(0).get(2));\n assertEquals(33.111, res.get(0).get(3));\n\n assertEquals(\"point-2d\", res.get(1).get(0));\n assertEquals(7203, res.get(1).get(1));\n assertEquals(22.0, res.get(1).get(2));\n assertEquals(44.222, res.get(1).get(3));\n }\n\n @Test\n public void testReadNodeWithGeoPointArray() {\n Dataset df = initTest(\"CREATE (p:Person {locations: [point({longitude: 11, latitude: 33.111}), point({longitude: 22, latitude: 44.222})]})\");\n\n List res = df.select(\"locations\").collectAsList().get(0).getList(0);\n assertEquals(\"point-2d\", res.get(0).get(0));\n assertEquals(4326, res.get(0).get(1));\n assertEquals(11.0, res.get(0).get(2));\n assertEquals(33.111, res.get(0).get(3));\n\n assertEquals(\"point-2d\", res.get(1).get(0));\n assertEquals(4326, res.get(1).get(1));\n assertEquals(22.0, res.get(1).get(2));\n assertEquals(44.222, res.get(1).get(3));\n }\n\n @Test\n public void testReadNodeWithPoint3DArray() {\n Dataset df = initTest(\"CREATE (p:Person {locations: [point({x: 11, y: 33.111, z: 12}), point({x: 22, y: 44.222, z: 99.1})]})\");\n\n List res = df.select(\"locations\").collectAsList().get(0).getList(0);\n assertEquals(\"point-3d\", res.get(0).get(0));\n assertEquals(9157, res.get(0).get(1));\n assertEquals(11.0, res.get(0).get(2));\n assertEquals(33.111, res.get(0).get(3));\n\n assertEquals(\"point-3d\", res.get(1).get(0));\n assertEquals(9157, res.get(1).get(1));\n assertEquals(22.0, res.get(1).get(2));\n assertEquals(44.222, res.get(1).get(3));\n }\n\n @Test\n public void testReadNodeWithMap() {\n Dataset df = ss().read().format(DataSource.class.getName())\n .option(\"url\", SparkConnectorScalaSuiteIT.server().getBoltUrl())\n .option(\"query\", \"RETURN {a: 1, b: '3'} AS map\")\n .load();\n\n Map map = df.select(\"map\").collectAsList().get(0).getJavaMap(0);\n Map expectedMap = new HashMap<>();\n expectedMap.put(\"a\", \"1\");\n expectedMap.put(\"b\", \"3\");\n\n assertEquals(expectedMap, map);\n }\n\n Dataset initTest(String query) {\n try (Session session = SparkConnectorScalaSuiteIT.session(\"\")) {\n session.writeTransaction(transaction -> transaction.run(query).consume());\n }\n\n return ss().read().format(DataSource.class.getName())\n .option(\"url\", SparkConnectorScalaSuiteIT.server().getBoltUrl())\n .option(\"labels\", \"Person\")\n .load();\n }\n}\n" }, { "path": "spark-3/src/test/java/org/neo4j/spark/SparkConnectorSuiteIT.java", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark;\n\nimport org.junit.runner.RunWith;\nimport org.junit.runners.Suite;\n\n@RunWith(Suite.class)\n@Suite.SuiteClasses({\n DataSourceReaderTypesTSE.class\n})\npublic class SparkConnectorSuiteIT extends SparkConnectorScalaSuiteIT {\n}\n" }, { "path": "spark-3/src/test/resources/log4j2.properties", "content": "appender.console.type=Console\nappender.console.name=STDOUT_TXT\nappender.console.target=SYSTEM_OUT\nappender.console.layout.type=PatternLayout\nappender.console.layout.pattern=%d{HH:mm:ss.SSS} [%t] %-5level %logger %X{source_type} - %msg%n\n\nlogger.neo4j.name=org.neo4j\nlogger.neo4j.level=debug\n\nrootLogger.level=info\nrootLogger.appenderRefs=stdout\nrootLogger.appenderRef.stdout.ref=STDOUT_TXT" }, { "path": "spark-3/src/test/resources/neo4j-sso-test-realm.json", "content": "{\n \"id\": \"7d10aebd-a60d-45a9-bce3-bb3a0a372ae9\",\n \"realm\": \"neo4j-sso-test\",\n \"notBefore\": 0,\n \"defaultSignatureAlgorithm\": \"RS256\",\n \"revokeRefreshToken\": false,\n \"refreshTokenMaxReuse\": 0,\n \"accessTokenLifespan\": 300,\n \"accessTokenLifespanForImplicitFlow\": 900,\n \"ssoSessionIdleTimeout\": 1800,\n \"ssoSessionMaxLifespan\": 36000,\n \"ssoSessionIdleTimeoutRememberMe\": 0,\n \"ssoSessionMaxLifespanRememberMe\": 0,\n \"offlineSessionIdleTimeout\": 2592000,\n \"offlineSessionMaxLifespanEnabled\": false,\n \"offlineSessionMaxLifespan\": 5184000,\n \"clientSessionIdleTimeout\": 0,\n \"clientSessionMaxLifespan\": 0,\n \"clientOfflineSessionIdleTimeout\": 0,\n \"clientOfflineSessionMaxLifespan\": 0,\n \"accessCodeLifespan\": 60,\n \"accessCodeLifespanUserAction\": 300,\n \"accessCodeLifespanLogin\": 1800,\n \"actionTokenGeneratedByAdminLifespan\": 43200,\n \"actionTokenGeneratedByUserLifespan\": 300,\n \"oauth2DeviceCodeLifespan\": 600,\n \"oauth2DevicePollingInterval\": 5,\n \"enabled\": true,\n \"sslRequired\": \"external\",\n \"registrationAllowed\": false,\n \"registrationEmailAsUsername\": false,\n \"rememberMe\": false,\n \"verifyEmail\": false,\n \"loginWithEmailAllowed\": true,\n \"duplicateEmailsAllowed\": false,\n \"resetPasswordAllowed\": false,\n \"editUsernameAllowed\": false,\n \"bruteForceProtected\": false,\n \"permanentLockout\": false,\n \"maxTemporaryLockouts\": 0,\n \"bruteForceStrategy\": \"MULTIPLE\",\n \"maxFailureWaitSeconds\": 900,\n \"minimumQuickLoginWaitSeconds\": 60,\n \"waitIncrementSeconds\": 60,\n \"quickLoginCheckMilliSeconds\": 1000,\n \"maxDeltaTimeSeconds\": 43200,\n \"failureFactor\": 30,\n \"roles\": {\n \"realm\": [\n {\n \"id\": \"4d08fd99-cc5d-45d6-9a49-cdd4fcd6448e\",\n \"name\": \"uma_authorization\",\n \"description\": \"${role_uma_authorization}\",\n \"composite\": false,\n \"clientRole\": false,\n \"containerId\": \"7d10aebd-a60d-45a9-bce3-bb3a0a372ae9\",\n \"attributes\": {}\n },\n {\n \"id\": \"49ad4e27-73f3-48ea-adf1-ff56880176d9\",\n \"name\": \"offline_access\",\n \"description\": \"${role_offline-access}\",\n \"composite\": false,\n \"clientRole\": false,\n \"containerId\": \"7d10aebd-a60d-45a9-bce3-bb3a0a372ae9\",\n \"attributes\": {}\n },\n {\n \"id\": \"1c26e8ea-3815-4364-b254-39280413343b\",\n \"name\": \"default-roles-neo4j-sso-test\",\n \"description\": \"${role_default-roles}\",\n \"composite\": true,\n \"composites\": {\n \"realm\": [\n \"offline_access\",\n \"uma_authorization\"\n ],\n \"client\": {\n \"account\": [\n \"manage-account\",\n \"view-profile\"\n ]\n }\n },\n \"clientRole\": false,\n \"containerId\": \"7d10aebd-a60d-45a9-bce3-bb3a0a372ae9\",\n \"attributes\": {}\n }\n ],\n \"client\": {\n \"realm-management\": [\n {\n \"id\": \"02f4e601-c963-4fdd-9246-f732ceb7ff40\",\n \"name\": \"manage-users\",\n \"description\": \"${role_manage-users}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"860945bd-3a1b-4f1d-91e4-6506ed427e72\",\n \"name\": \"query-clients\",\n \"description\": \"${role_query-clients}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"0b3a6477-57be-4ebe-88d0-473f9bbcaa43\",\n \"name\": \"query-realms\",\n \"description\": \"${role_query-realms}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"07e63113-153b-4ca1-95c7-a0c161b6107a\",\n \"name\": \"realm-admin\",\n \"description\": \"${role_realm-admin}\",\n \"composite\": true,\n \"composites\": {\n \"client\": {\n \"realm-management\": [\n \"manage-users\",\n \"query-clients\",\n \"query-realms\",\n \"view-clients\",\n \"query-users\",\n \"view-users\",\n \"manage-clients\",\n \"view-events\",\n \"query-groups\",\n \"manage-realm\",\n \"manage-authorization\",\n \"create-client\",\n \"manage-events\",\n \"view-realm\",\n \"manage-identity-providers\",\n \"impersonation\",\n \"view-authorization\",\n \"view-identity-providers\"\n ]\n }\n },\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"89500893-787c-47a0-8e54-917d54508cff\",\n \"name\": \"query-users\",\n \"description\": \"${role_query-users}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"29194689-d523-40ef-9340-88eb86c6c4bb\",\n \"name\": \"view-clients\",\n \"description\": \"${role_view-clients}\",\n \"composite\": true,\n \"composites\": {\n \"client\": {\n \"realm-management\": [\n \"query-clients\"\n ]\n }\n },\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"e871e84c-d6d2-4c3f-bd7b-b9a5aeac32be\",\n \"name\": \"manage-clients\",\n \"description\": \"${role_manage-clients}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"8f862a23-e90d-4f92-8acf-fb9adb5121f0\",\n \"name\": \"view-users\",\n \"description\": \"${role_view-users}\",\n \"composite\": true,\n \"composites\": {\n \"client\": {\n \"realm-management\": [\n \"query-groups\",\n \"query-users\"\n ]\n }\n },\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"f36e6ae9-043d-47ee-b294-2f450d0f34a9\",\n \"name\": \"manage-realm\",\n \"description\": \"${role_manage-realm}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"802b1f48-95cc-4af5-bc96-76d8a701ec24\",\n \"name\": \"query-groups\",\n \"description\": \"${role_query-groups}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"743ede07-a12d-4d6b-bec2-0231450a28f9\",\n \"name\": \"view-events\",\n \"description\": \"${role_view-events}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"7252234e-695b-47a2-bfa9-7c50b5bedd35\",\n \"name\": \"manage-authorization\",\n \"description\": \"${role_manage-authorization}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"3d5a003d-3c0d-4193-9048-40179baa1f5f\",\n \"name\": \"create-client\",\n \"description\": \"${role_create-client}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"174cb3a3-b04a-446b-88d8-d5225a658757\",\n \"name\": \"manage-events\",\n \"description\": \"${role_manage-events}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"1596f494-4617-4794-ba44-d7add890691e\",\n \"name\": \"view-realm\",\n \"description\": \"${role_view-realm}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"daf38dba-70c6-4a38-9a56-92d42bcc4731\",\n \"name\": \"impersonation\",\n \"description\": \"${role_impersonation}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"64ea62ef-c69c-400f-af0d-f63f410a5801\",\n \"name\": \"manage-identity-providers\",\n \"description\": \"${role_manage-identity-providers}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"f6189b24-7b3b-4acf-8240-5297b60a82e5\",\n \"name\": \"view-authorization\",\n \"description\": \"${role_view-authorization}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n },\n {\n \"id\": \"ef2dea85-dd33-4aed-932b-a8e8bd2355ca\",\n \"name\": \"view-identity-providers\",\n \"description\": \"${role_view-identity-providers}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"attributes\": {}\n }\n ],\n \"security-admin-console\": [],\n \"admin-cli\": [],\n \"neo4j-commons-client\": [\n {\n \"id\": \"9a142f12-99da-442c-bdaa-9f8a59171404\",\n \"name\": \"uma_protection\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"004fd711-5103-47bc-9c93-bfcf38b3a10a\",\n \"attributes\": {}\n }\n ],\n \"account-console\": [],\n \"broker\": [\n {\n \"id\": \"06e885bf-8d8f-4a7b-afde-4bf4a87959d3\",\n \"name\": \"read-token\",\n \"description\": \"${role_read-token}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"7c1f30ad-9cfd-4817-9f02-7f914b125de7\",\n \"attributes\": {}\n }\n ],\n \"account\": [\n {\n \"id\": \"1f367fdb-6d7a-4c7f-af5b-a0ed66b1c289\",\n \"name\": \"delete-account\",\n \"description\": \"${role_delete-account}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"f78d45f3-369c-457e-ab92-4e1e5a97f54c\",\n \"attributes\": {}\n },\n {\n \"id\": \"3fad9c59-832e-48d4-abc8-82a621a66ce9\",\n \"name\": \"manage-account\",\n \"description\": \"${role_manage-account}\",\n \"composite\": true,\n \"composites\": {\n \"client\": {\n \"account\": [\n \"manage-account-links\"\n ]\n }\n },\n \"clientRole\": true,\n \"containerId\": \"f78d45f3-369c-457e-ab92-4e1e5a97f54c\",\n \"attributes\": {}\n },\n {\n \"id\": \"1d0cc167-ae58-4950-97e8-9f709ee118a6\",\n \"name\": \"view-consent\",\n \"description\": \"${role_view-consent}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"f78d45f3-369c-457e-ab92-4e1e5a97f54c\",\n \"attributes\": {}\n },\n {\n \"id\": \"1d83e541-8475-48e1-afd4-6fa17429275c\",\n \"name\": \"manage-account-links\",\n \"description\": \"${role_manage-account-links}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"f78d45f3-369c-457e-ab92-4e1e5a97f54c\",\n \"attributes\": {}\n },\n {\n \"id\": \"4e1703b0-d6f2-4504-aa4e-8670b14e88fe\",\n \"name\": \"view-applications\",\n \"description\": \"${role_view-applications}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"f78d45f3-369c-457e-ab92-4e1e5a97f54c\",\n \"attributes\": {}\n },\n {\n \"id\": \"cd8d0800-a6b3-43fc-bd15-0c7ce40c9183\",\n \"name\": \"view-profile\",\n \"description\": \"${role_view-profile}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"f78d45f3-369c-457e-ab92-4e1e5a97f54c\",\n \"attributes\": {}\n },\n {\n \"id\": \"62439512-e52e-41ef-9123-66d79931f45b\",\n \"name\": \"manage-consent\",\n \"description\": \"${role_manage-consent}\",\n \"composite\": true,\n \"composites\": {\n \"client\": {\n \"account\": [\n \"view-consent\"\n ]\n }\n },\n \"clientRole\": true,\n \"containerId\": \"f78d45f3-369c-457e-ab92-4e1e5a97f54c\",\n \"attributes\": {}\n },\n {\n \"id\": \"d67ceb33-f9d2-482f-99c4-09a29210ee78\",\n \"name\": \"view-groups\",\n \"description\": \"${role_view-groups}\",\n \"composite\": false,\n \"clientRole\": true,\n \"containerId\": \"f78d45f3-369c-457e-ab92-4e1e5a97f54c\",\n \"attributes\": {}\n }\n ]\n }\n },\n \"groups\": [\n {\n \"id\": \"667caafa-3dbe-4604-a427-6c61ecac5c3f\",\n \"name\": \"admin\",\n \"path\": \"/admin\",\n \"subGroups\": [],\n \"attributes\": {},\n \"realmRoles\": [],\n \"clientRoles\": {}\n }\n ],\n \"defaultRole\": {\n \"id\": \"1c26e8ea-3815-4364-b254-39280413343b\",\n \"name\": \"default-roles-neo4j-sso-test\",\n \"description\": \"${role_default-roles}\",\n \"composite\": true,\n \"clientRole\": false,\n \"containerId\": \"7d10aebd-a60d-45a9-bce3-bb3a0a372ae9\"\n },\n \"requiredCredentials\": [\n \"password\"\n ],\n \"otpPolicyType\": \"totp\",\n \"otpPolicyAlgorithm\": \"HmacSHA1\",\n \"otpPolicyInitialCounter\": 0,\n \"otpPolicyDigits\": 6,\n \"otpPolicyLookAheadWindow\": 1,\n \"otpPolicyPeriod\": 30,\n \"otpPolicyCodeReusable\": false,\n \"otpSupportedApplications\": [\n \"totpAppFreeOTPName\",\n \"totpAppGoogleName\",\n \"totpAppMicrosoftAuthenticatorName\"\n ],\n \"localizationTexts\": {},\n \"webAuthnPolicyRpEntityName\": \"keycloak\",\n \"webAuthnPolicySignatureAlgorithms\": [\n \"ES256\",\n \"RS256\"\n ],\n \"webAuthnPolicyRpId\": \"\",\n \"webAuthnPolicyAttestationConveyancePreference\": \"not specified\",\n \"webAuthnPolicyAuthenticatorAttachment\": \"not specified\",\n \"webAuthnPolicyRequireResidentKey\": \"not specified\",\n \"webAuthnPolicyUserVerificationRequirement\": \"not specified\",\n \"webAuthnPolicyCreateTimeout\": 0,\n \"webAuthnPolicyAvoidSameAuthenticatorRegister\": false,\n \"webAuthnPolicyAcceptableAaguids\": [],\n \"webAuthnPolicyExtraOrigins\": [],\n \"webAuthnPolicyPasswordlessRpEntityName\": \"keycloak\",\n \"webAuthnPolicyPasswordlessSignatureAlgorithms\": [\n \"ES256\",\n \"RS256\"\n ],\n \"webAuthnPolicyPasswordlessRpId\": \"\",\n \"webAuthnPolicyPasswordlessAttestationConveyancePreference\": \"not specified\",\n \"webAuthnPolicyPasswordlessAuthenticatorAttachment\": \"not specified\",\n \"webAuthnPolicyPasswordlessRequireResidentKey\": \"not specified\",\n \"webAuthnPolicyPasswordlessUserVerificationRequirement\": \"not specified\",\n \"webAuthnPolicyPasswordlessCreateTimeout\": 0,\n \"webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister\": false,\n \"webAuthnPolicyPasswordlessAcceptableAaguids\": [],\n \"webAuthnPolicyPasswordlessExtraOrigins\": [],\n \"users\" : [\n {\n \"id\" : \"14236e67-7ec2-490f-9f25-9e14af69e52a\",\n \"username\" : \"john-tester\",\n \"firstName\" : \"John\",\n \"lastName\" : \"Tester\",\n \"email\" : \"john.tester@test.com\",\n \"emailVerified\" : false,\n \"createdTimestamp\" : 1753905058600,\n \"enabled\" : true,\n \"totp\" : false,\n \"credentials\" : [ {\n \"id\" : \"d0fa2175-584f-45ec-a242-283b54a84feb\",\n \"type\" : \"password\",\n \"userLabel\" : \"My password\",\n \"createdDate\" : 1753905074445,\n \"secretData\" : \"{\\\"value\\\":\\\"XvUWimFjFwz3KoXoChpHcbypIlm/I1K/s4nYHNHQfMU=\\\",\\\"salt\\\":\\\"yEQApqyMC2yQCdU/7uLJzQ==\\\",\\\"additionalParameters\\\":{}}\",\n \"credentialData\" : \"{\\\"hashIterations\\\":5,\\\"algorithm\\\":\\\"argon2\\\",\\\"additionalParameters\\\":{\\\"hashLength\\\":[\\\"32\\\"],\\\"memory\\\":[\\\"7168\\\"],\\\"type\\\":[\\\"id\\\"],\\\"version\\\":[\\\"1.3\\\"],\\\"parallelism\\\":[\\\"1\\\"]}}\"\n } ],\n \"disableableCredentialTypes\" : [ ],\n \"requiredActions\" : [ ],\n \"realmRoles\" : [ \"default-roles-neo4j-sso-test\" ],\n \"notBefore\" : 0,\n \"groups\" : [ \"/admin\" ]\n }, {\n \"id\" : \"61144f9e-1bff-4b3e-beeb-a04d3e0cbe1c\",\n \"username\" : \"service-account-neo4j-commons-client\",\n \"emailVerified\" : false,\n \"createdTimestamp\" : 1753904616478,\n \"enabled\" : true,\n \"totp\" : false,\n \"serviceAccountClientId\" : \"neo4j-commons-client\",\n \"credentials\" : [ ],\n \"disableableCredentialTypes\" : [ ],\n \"requiredActions\" : [ ],\n \"realmRoles\" : [ \"default-roles-neo4j-sso-test\" ],\n \"clientRoles\" : {\n \"neo4j-commons-client\" : [ \"uma_protection\" ]\n },\n \"notBefore\" : 0,\n \"groups\" : [ ]\n } ],\n \"scopeMappings\": [\n {\n \"clientScope\": \"offline_access\",\n \"roles\": [\n \"offline_access\"\n ]\n }\n ],\n \"clientScopeMappings\": {\n \"account\": [\n {\n \"client\": \"account-console\",\n \"roles\": [\n \"manage-account\",\n \"view-groups\"\n ]\n }\n ]\n },\n \"clients\": [\n {\n \"id\": \"f78d45f3-369c-457e-ab92-4e1e5a97f54c\",\n \"clientId\": \"account\",\n \"name\": \"${client_account}\",\n \"rootUrl\": \"${authBaseUrl}\",\n \"baseUrl\": \"/realms/neo4j-sso-test/account/\",\n \"surrogateAuthRequired\": false,\n \"enabled\": true,\n \"alwaysDisplayInConsole\": false,\n \"clientAuthenticatorType\": \"client-secret\",\n \"redirectUris\": [\n \"/realms/neo4j-sso-test/account/*\"\n ],\n \"webOrigins\": [],\n \"notBefore\": 0,\n \"bearerOnly\": false,\n \"consentRequired\": false,\n \"standardFlowEnabled\": true,\n \"implicitFlowEnabled\": false,\n \"directAccessGrantsEnabled\": false,\n \"serviceAccountsEnabled\": false,\n \"publicClient\": true,\n \"frontchannelLogout\": false,\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"realm_client\": \"false\",\n \"post.logout.redirect.uris\": \"+\"\n },\n \"authenticationFlowBindingOverrides\": {},\n \"fullScopeAllowed\": false,\n \"nodeReRegistrationTimeout\": 0,\n \"defaultClientScopes\": [\n \"web-origins\",\n \"acr\",\n \"profile\",\n \"roles\",\n \"basic\",\n \"email\"\n ],\n \"optionalClientScopes\": [\n \"address\",\n \"phone\",\n \"organization\",\n \"offline_access\",\n \"microprofile-jwt\"\n ]\n },\n {\n \"id\": \"60b52f12-1c88-418b-8de5-181f69a7f12f\",\n \"clientId\": \"account-console\",\n \"name\": \"${client_account-console}\",\n \"rootUrl\": \"${authBaseUrl}\",\n \"baseUrl\": \"/realms/neo4j-sso-test/account/\",\n \"surrogateAuthRequired\": false,\n \"enabled\": true,\n \"alwaysDisplayInConsole\": false,\n \"clientAuthenticatorType\": \"client-secret\",\n \"redirectUris\": [\n \"/realms/neo4j-sso-test/account/*\"\n ],\n \"webOrigins\": [],\n \"notBefore\": 0,\n \"bearerOnly\": false,\n \"consentRequired\": false,\n \"standardFlowEnabled\": true,\n \"implicitFlowEnabled\": false,\n \"directAccessGrantsEnabled\": false,\n \"serviceAccountsEnabled\": false,\n \"publicClient\": true,\n \"frontchannelLogout\": false,\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"realm_client\": \"false\",\n \"post.logout.redirect.uris\": \"+\",\n \"pkce.code.challenge.method\": \"S256\"\n },\n \"authenticationFlowBindingOverrides\": {},\n \"fullScopeAllowed\": false,\n \"nodeReRegistrationTimeout\": 0,\n \"protocolMappers\": [\n {\n \"id\": \"7281166b-a81b-485d-a227-b47299ea7a10\",\n \"name\": \"audience resolve\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-audience-resolve-mapper\",\n \"consentRequired\": false,\n \"config\": {}\n }\n ],\n \"defaultClientScopes\": [\n \"web-origins\",\n \"acr\",\n \"profile\",\n \"roles\",\n \"basic\",\n \"email\"\n ],\n \"optionalClientScopes\": [\n \"address\",\n \"phone\",\n \"organization\",\n \"offline_access\",\n \"microprofile-jwt\"\n ]\n },\n {\n \"id\": \"2c20b6a9-7bb6-4d41-b1bb-1375de214d5a\",\n \"clientId\": \"admin-cli\",\n \"name\": \"${client_admin-cli}\",\n \"surrogateAuthRequired\": false,\n \"enabled\": true,\n \"alwaysDisplayInConsole\": false,\n \"clientAuthenticatorType\": \"client-secret\",\n \"redirectUris\": [],\n \"webOrigins\": [],\n \"notBefore\": 0,\n \"bearerOnly\": false,\n \"consentRequired\": false,\n \"standardFlowEnabled\": false,\n \"implicitFlowEnabled\": false,\n \"directAccessGrantsEnabled\": true,\n \"serviceAccountsEnabled\": false,\n \"publicClient\": true,\n \"frontchannelLogout\": false,\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"realm_client\": \"false\",\n \"client.use.lightweight.access.token.enabled\": \"true\",\n \"post.logout.redirect.uris\": \"+\"\n },\n \"authenticationFlowBindingOverrides\": {},\n \"fullScopeAllowed\": true,\n \"nodeReRegistrationTimeout\": 0,\n \"defaultClientScopes\": [\n \"web-origins\",\n \"acr\",\n \"profile\",\n \"roles\",\n \"basic\",\n \"email\"\n ],\n \"optionalClientScopes\": [\n \"address\",\n \"phone\",\n \"organization\",\n \"offline_access\",\n \"microprofile-jwt\"\n ]\n },\n {\n \"id\": \"7c1f30ad-9cfd-4817-9f02-7f914b125de7\",\n \"clientId\": \"broker\",\n \"name\": \"${client_broker}\",\n \"surrogateAuthRequired\": false,\n \"enabled\": true,\n \"alwaysDisplayInConsole\": false,\n \"clientAuthenticatorType\": \"client-secret\",\n \"redirectUris\": [],\n \"webOrigins\": [],\n \"notBefore\": 0,\n \"bearerOnly\": true,\n \"consentRequired\": false,\n \"standardFlowEnabled\": true,\n \"implicitFlowEnabled\": false,\n \"directAccessGrantsEnabled\": false,\n \"serviceAccountsEnabled\": false,\n \"publicClient\": false,\n \"frontchannelLogout\": false,\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"realm_client\": \"true\",\n \"post.logout.redirect.uris\": \"+\"\n },\n \"authenticationFlowBindingOverrides\": {},\n \"fullScopeAllowed\": false,\n \"nodeReRegistrationTimeout\": 0,\n \"defaultClientScopes\": [\n \"web-origins\",\n \"acr\",\n \"profile\",\n \"roles\",\n \"basic\",\n \"email\"\n ],\n \"optionalClientScopes\": [\n \"address\",\n \"phone\",\n \"organization\",\n \"offline_access\",\n \"microprofile-jwt\"\n ]\n },\n {\n \"id\": \"004fd711-5103-47bc-9c93-bfcf38b3a10a\",\n \"clientId\": \"neo4j-commons-client\",\n \"name\": \"neo4j-commons-client\",\n \"description\": \"\",\n \"rootUrl\": \"\",\n \"adminUrl\": \"\",\n \"baseUrl\": \"\",\n \"surrogateAuthRequired\": false,\n \"enabled\": true,\n \"alwaysDisplayInConsole\": false,\n \"clientAuthenticatorType\": \"client-secret\",\n \"secret\": \"QNrSpbh0mxhnlYlI21UcBaz3Htb734vi\",\n \"redirectUris\": [\n \"/*\"\n ],\n \"webOrigins\": [\n \"/*\"\n ],\n \"notBefore\": 0,\n \"bearerOnly\": false,\n \"consentRequired\": false,\n \"standardFlowEnabled\": true,\n \"implicitFlowEnabled\": false,\n \"directAccessGrantsEnabled\": true,\n \"serviceAccountsEnabled\": true,\n \"authorizationServicesEnabled\": true,\n \"publicClient\": false,\n \"frontchannelLogout\": true,\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"access.token.lifespan\" : \"3\",\n \"client.secret.creation.time\" : \"1750236324\",\n \"request.object.signature.alg\" : \"any\",\n \"request.object.encryption.alg\" : \"any\",\n \"client.introspection.response.allow.jwt.claim.enabled\" : \"true\",\n \"standard.token.exchange.enabled\" : \"false\",\n \"frontchannel.logout.session.required\" : \"true\",\n \"oauth2.device.authorization.grant.enabled\" : \"true\",\n \"use.jwks.url\" : \"false\",\n \"backchannel.logout.revoke.offline.tokens\" : \"false\",\n \"use.refresh.tokens\" : \"true\",\n \"realm_client\" : \"false\",\n \"oidc.ciba.grant.enabled\" : \"false\",\n \"client.use.lightweight.access.token.enabled\" : \"false\",\n \"backchannel.logout.session.required\" : \"true\",\n \"request.object.required\" : \"not required\",\n \"client_credentials.use_refresh_token\" : \"true\",\n \"access.token.header.type.rfc9068\" : \"false\",\n \"tls.client.certificate.bound.access.tokens\" : \"false\",\n \"require.pushed.authorization.requests\" : \"false\",\n \"acr.loa.map\" : \"{}\",\n \"display.on.consent.screen\" : \"false\",\n \"request.object.encryption.enc\" : \"any\",\n \"token.response.type.bearer.lower-case\" : \"false\"\n },\n \"authenticationFlowBindingOverrides\": {},\n \"fullScopeAllowed\": true,\n \"nodeReRegistrationTimeout\": -1,\n \"protocolMappers\" : [ {\n \"id\" : \"ac56f3e2-8898-4040-a4ed-b1a4e3a20e52\",\n \"name\" : \"groups\",\n \"protocol\" : \"openid-connect\",\n \"protocolMapper\" : \"oidc-group-membership-mapper\",\n \"consentRequired\" : false,\n \"config\" : {\n \"full.path\" : \"false\",\n \"introspection.token.claim\" : \"true\",\n \"userinfo.token.claim\" : \"false\",\n \"multivalued\" : \"true\",\n \"id.token.claim\" : \"false\",\n \"lightweight.claim\" : \"false\",\n \"access.token.claim\" : \"true\",\n \"claim.name\" : \"groups\"\n }\n } ],\n \"defaultClientScopes\": [\n \"service_account\",\n \"web-origins\",\n \"acr\",\n \"profile\",\n \"roles\",\n \"basic\",\n \"email\"\n ],\n \"optionalClientScopes\": [\n \"address\",\n \"phone\",\n \"organization\",\n \"offline_access\",\n \"microprofile-jwt\"\n ]\n },\n {\n \"id\": \"ff825411-d253-459a-93b9-71b9f00b1276\",\n \"clientId\": \"realm-management\",\n \"name\": \"${client_realm-management}\",\n \"surrogateAuthRequired\": false,\n \"enabled\": true,\n \"alwaysDisplayInConsole\": false,\n \"clientAuthenticatorType\": \"client-secret\",\n \"redirectUris\": [],\n \"webOrigins\": [],\n \"notBefore\": 0,\n \"bearerOnly\": true,\n \"consentRequired\": false,\n \"standardFlowEnabled\": true,\n \"implicitFlowEnabled\": false,\n \"directAccessGrantsEnabled\": false,\n \"serviceAccountsEnabled\": false,\n \"publicClient\": false,\n \"frontchannelLogout\": false,\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"realm_client\": \"true\",\n \"post.logout.redirect.uris\": \"+\"\n },\n \"authenticationFlowBindingOverrides\": {},\n \"fullScopeAllowed\": false,\n \"nodeReRegistrationTimeout\": 0,\n \"defaultClientScopes\": [\n \"web-origins\",\n \"acr\",\n \"profile\",\n \"roles\",\n \"basic\",\n \"email\"\n ],\n \"optionalClientScopes\": [\n \"address\",\n \"phone\",\n \"organization\",\n \"offline_access\",\n \"microprofile-jwt\"\n ]\n },\n {\n \"id\": \"e45c1497-2c8f-47f2-8c11-42ad8c088e23\",\n \"clientId\": \"security-admin-console\",\n \"name\": \"${client_security-admin-console}\",\n \"rootUrl\": \"${authAdminUrl}\",\n \"baseUrl\": \"/admin/neo4j-sso-test/console/\",\n \"surrogateAuthRequired\": false,\n \"enabled\": true,\n \"alwaysDisplayInConsole\": false,\n \"clientAuthenticatorType\": \"client-secret\",\n \"redirectUris\": [\n \"/admin/neo4j-sso-test/console/*\"\n ],\n \"webOrigins\": [\n \"+\"\n ],\n \"notBefore\": 0,\n \"bearerOnly\": false,\n \"consentRequired\": false,\n \"standardFlowEnabled\": true,\n \"implicitFlowEnabled\": false,\n \"directAccessGrantsEnabled\": false,\n \"serviceAccountsEnabled\": false,\n \"publicClient\": true,\n \"frontchannelLogout\": false,\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"realm_client\": \"false\",\n \"client.use.lightweight.access.token.enabled\": \"true\",\n \"post.logout.redirect.uris\": \"+\",\n \"pkce.code.challenge.method\": \"S256\"\n },\n \"authenticationFlowBindingOverrides\": {},\n \"fullScopeAllowed\": true,\n \"nodeReRegistrationTimeout\": 0,\n \"protocolMappers\": [\n {\n \"id\": \"eb064310-4c24-4c0f-bf39-27d5767a2f21\",\n \"name\": \"locale\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"locale\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"locale\",\n \"jsonType.label\": \"String\"\n }\n }\n ],\n \"defaultClientScopes\": [\n \"web-origins\",\n \"acr\",\n \"profile\",\n \"roles\",\n \"basic\",\n \"email\"\n ],\n \"optionalClientScopes\": [\n \"address\",\n \"phone\",\n \"organization\",\n \"offline_access\",\n \"microprofile-jwt\"\n ]\n }\n ],\n \"clientScopes\": [\n {\n \"id\": \"6daac52d-0f54-4cbb-b159-93cae1188e26\",\n \"name\": \"service_account\",\n \"description\": \"Specific scope for a client enabled for service accounts\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"include.in.token.scope\": \"false\",\n \"display.on.consent.screen\": \"false\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"a7d31bbd-ca7b-4d61-ae3a-7d901eb84039\",\n \"name\": \"Client Host\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usersessionmodel-note-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"user.session.note\": \"clientHost\",\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"clientHost\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"8c417a42-c005-46ff-8e30-834ca3170169\",\n \"name\": \"Client ID\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usersessionmodel-note-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"user.session.note\": \"client_id\",\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"client_id\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"3d484d15-8210-417d-af09-5cdedab9c6da\",\n \"name\": \"Client IP Address\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usersessionmodel-note-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"user.session.note\": \"clientAddress\",\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"clientAddress\",\n \"jsonType.label\": \"String\"\n }\n }\n ]\n },\n {\n \"id\": \"93bbac6c-0437-46ad-8917-29b8f9a03376\",\n \"name\": \"phone\",\n \"description\": \"OpenID Connect built-in scope: phone\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"include.in.token.scope\": \"true\",\n \"consent.screen.text\": \"${phoneScopeConsentText}\",\n \"display.on.consent.screen\": \"true\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"6695fac9-a5a7-4705-aadb-c57a09626042\",\n \"name\": \"phone number verified\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"phoneNumberVerified\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"phone_number_verified\",\n \"jsonType.label\": \"boolean\"\n }\n },\n {\n \"id\": \"c5f8b96f-7586-4468-837a-d94f2eff1f37\",\n \"name\": \"phone number\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"phoneNumber\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"phone_number\",\n \"jsonType.label\": \"String\"\n }\n }\n ]\n },\n {\n \"id\": \"0430729b-4f3e-44ce-8d2c-1eec8e077357\",\n \"name\": \"saml_organization\",\n \"description\": \"Organization Membership\",\n \"protocol\": \"saml\",\n \"attributes\": {\n \"display.on.consent.screen\": \"false\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"d5918bc9-1660-402f-8952-92571a955835\",\n \"name\": \"organization\",\n \"protocol\": \"saml\",\n \"protocolMapper\": \"saml-organization-membership-mapper\",\n \"consentRequired\": false,\n \"config\": {}\n }\n ]\n },\n {\n \"id\": \"aab0e3e4-5f85-4015-802d-d5286e9ad947\",\n \"name\": \"profile\",\n \"description\": \"OpenID Connect built-in scope: profile\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"include.in.token.scope\": \"true\",\n \"consent.screen.text\": \"${profileScopeConsentText}\",\n \"display.on.consent.screen\": \"true\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"75f3fd4d-04b6-4bc0-876b-07f598dbb669\",\n \"name\": \"family name\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"lastName\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"family_name\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"7e619c16-d618-4f7f-bc46-4ba4fc0d8f7b\",\n \"name\": \"full name\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-full-name-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"id.token.claim\": \"true\",\n \"introspection.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\"\n }\n },\n {\n \"id\": \"8476d1e0-9dcc-4318-9a58-6f52d82a8b46\",\n \"name\": \"locale\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"locale\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"locale\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"09e15aea-9ce2-420f-baac-76f358b8ac5b\",\n \"name\": \"username\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"username\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"preferred_username\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"faf6d7d0-3418-44b4-ba41-5b6bfd105fdb\",\n \"name\": \"gender\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"gender\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"gender\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"710d18f8-2c62-4909-b0ea-99975876e2ae\",\n \"name\": \"middle name\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"middleName\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"middle_name\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"1871201c-4d1e-4893-8775-3bd4ee6ee4ab\",\n \"name\": \"picture\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"picture\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"picture\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"66d85a48-55c0-491d-b05a-9ba75093a6ab\",\n \"name\": \"updated at\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"updatedAt\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"updated_at\",\n \"jsonType.label\": \"long\"\n }\n },\n {\n \"id\": \"130c0094-a76b-4235-a6a1-45fdae43b2ba\",\n \"name\": \"nickname\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"nickname\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"nickname\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"8100f44e-0052-4568-8c8c-3964c2da1cfd\",\n \"name\": \"profile\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"profile\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"profile\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"c7d9c5e2-8b82-4337-853a-b079c4d40cd4\",\n \"name\": \"given name\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"firstName\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"given_name\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"cb9bb72d-3cfd-4ae7-98d1-567a132afbeb\",\n \"name\": \"birthdate\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"birthdate\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"birthdate\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"713625e4-e1cb-4ba0-9339-983e6038fb7d\",\n \"name\": \"website\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"website\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"website\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"5d44e729-70f0-41a0-b22b-2f51915accae\",\n \"name\": \"zoneinfo\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"zoneinfo\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"zoneinfo\",\n \"jsonType.label\": \"String\"\n }\n }\n ]\n },\n {\n \"id\": \"655a4881-b5f1-4844-a60a-e5974b6dede9\",\n \"name\": \"email\",\n \"description\": \"OpenID Connect built-in scope: email\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"include.in.token.scope\": \"true\",\n \"consent.screen.text\": \"${emailScopeConsentText}\",\n \"display.on.consent.screen\": \"true\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"1a203587-27fb-4c64-b1b2-6603bfc29a19\",\n \"name\": \"email verified\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-property-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"emailVerified\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"email_verified\",\n \"jsonType.label\": \"boolean\"\n }\n },\n {\n \"id\": \"db51d38a-be10-40c1-a054-6124b17711c7\",\n \"name\": \"email\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"email\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"email\",\n \"jsonType.label\": \"String\"\n }\n }\n ]\n },\n {\n \"id\": \"a2ebcfef-8d3f-493d-9deb-c0bbe0feffb4\",\n \"name\": \"acr\",\n \"description\": \"OpenID Connect scope for add acr (authentication context class reference) to the token\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"include.in.token.scope\": \"false\",\n \"display.on.consent.screen\": \"false\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"58b572de-ed96-45d4-8b64-3193c03e8e00\",\n \"name\": \"acr loa level\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-acr-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"id.token.claim\": \"true\",\n \"introspection.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\"\n }\n }\n ]\n },\n {\n \"id\": \"caa43308-3e68-4a0b-866a-ec350687d020\",\n \"name\": \"roles\",\n \"description\": \"OpenID Connect scope for add user roles to the access token\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"include.in.token.scope\": \"false\",\n \"consent.screen.text\": \"${rolesScopeConsentText}\",\n \"display.on.consent.screen\": \"true\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"263ca184-249e-40a5-88e6-e7062ffc416d\",\n \"name\": \"realm roles\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-realm-role-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"user.attribute\": \"foo\",\n \"introspection.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"realm_access.roles\",\n \"jsonType.label\": \"String\",\n \"multivalued\": \"true\"\n }\n },\n {\n \"id\": \"e08c2804-bae8-4622-892f-bf002894d804\",\n \"name\": \"client roles\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-client-role-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"user.attribute\": \"foo\",\n \"introspection.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"resource_access.${client_id}.roles\",\n \"jsonType.label\": \"String\",\n \"multivalued\": \"true\"\n }\n },\n {\n \"id\": \"ae8f0f35-d7a1-4f4d-94b9-a167c231d75b\",\n \"name\": \"audience resolve\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-audience-resolve-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"access.token.claim\": \"true\"\n }\n }\n ]\n },\n {\n \"id\": \"22c7fd6a-a90d-4482-85d3-dc5a51062bd4\",\n \"name\": \"organization\",\n \"description\": \"Additional claims about the organization a subject belongs to\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"include.in.token.scope\": \"true\",\n \"consent.screen.text\": \"${organizationScopeConsentText}\",\n \"display.on.consent.screen\": \"true\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"79e6ae6c-c710-4ccf-a9d7-c1e8e8836d6a\",\n \"name\": \"organization\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-organization-membership-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"multivalued\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"organization\",\n \"jsonType.label\": \"String\"\n }\n }\n ]\n },\n {\n \"id\": \"c85b8ffe-1a80-4012-9642-ddf25d112c89\",\n \"name\": \"basic\",\n \"description\": \"OpenID Connect scope for add all basic claims to the token\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"include.in.token.scope\": \"false\",\n \"display.on.consent.screen\": \"false\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"d7f80a87-6368-47de-b8af-9fec687114fa\",\n \"name\": \"auth_time\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usersessionmodel-note-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"user.session.note\": \"AUTH_TIME\",\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"auth_time\",\n \"jsonType.label\": \"long\"\n }\n },\n {\n \"id\": \"2b319ed3-d20f-407c-9cbd-c5478ef4a9d1\",\n \"name\": \"sub\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-sub-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"access.token.claim\": \"true\"\n }\n }\n ]\n },\n {\n \"id\": \"2942a349-f522-43cc-b983-e1b2119af9db\",\n \"name\": \"role_list\",\n \"description\": \"SAML role list\",\n \"protocol\": \"saml\",\n \"attributes\": {\n \"consent.screen.text\": \"${samlRoleListScopeConsentText}\",\n \"display.on.consent.screen\": \"true\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"40a89e81-e9eb-463d-b5dc-b124e64ffa57\",\n \"name\": \"role list\",\n \"protocol\": \"saml\",\n \"protocolMapper\": \"saml-role-list-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"single\": \"false\",\n \"attribute.nameformat\": \"Basic\",\n \"attribute.name\": \"Role\"\n }\n }\n ]\n },\n {\n \"id\": \"0af3471d-ddde-4e01-9f64-137725d70416\",\n \"name\": \"offline_access\",\n \"description\": \"OpenID Connect built-in scope: offline_access\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"consent.screen.text\": \"${offlineAccessScopeConsentText}\",\n \"display.on.consent.screen\": \"true\"\n }\n },\n {\n \"id\": \"a600ed15-1dc0-4c45-8262-1bd34bba9ca5\",\n \"name\": \"microprofile-jwt\",\n \"description\": \"Microprofile - JWT built-in scope\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"include.in.token.scope\": \"true\",\n \"display.on.consent.screen\": \"false\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"095068ea-7f54-4bf6-9cf2-331ea30368dc\",\n \"name\": \"upn\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-attribute-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"username\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"upn\",\n \"jsonType.label\": \"String\"\n }\n },\n {\n \"id\": \"0e12f986-9b38-475c-a18a-2da1e95f4831\",\n \"name\": \"groups\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-usermodel-realm-role-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"multivalued\": \"true\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute\": \"foo\",\n \"id.token.claim\": \"true\",\n \"access.token.claim\": \"true\",\n \"claim.name\": \"groups\",\n \"jsonType.label\": \"String\"\n }\n }\n ]\n },\n {\n \"id\": \"6235a6e8-21ee-4498-9c95-b3816efe0dfc\",\n \"name\": \"address\",\n \"description\": \"OpenID Connect built-in scope: address\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"include.in.token.scope\": \"true\",\n \"consent.screen.text\": \"${addressScopeConsentText}\",\n \"display.on.consent.screen\": \"true\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"1b5f1955-723a-4ec5-a8eb-578fb9e9ebb0\",\n \"name\": \"address\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-address-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"user.attribute.formatted\": \"formatted\",\n \"user.attribute.country\": \"country\",\n \"introspection.token.claim\": \"true\",\n \"user.attribute.postal_code\": \"postal_code\",\n \"userinfo.token.claim\": \"true\",\n \"user.attribute.street\": \"street\",\n \"id.token.claim\": \"true\",\n \"user.attribute.region\": \"region\",\n \"access.token.claim\": \"true\",\n \"user.attribute.locality\": \"locality\"\n }\n }\n ]\n },\n {\n \"id\": \"a9966629-729a-4d02-931c-cdbccc2ef6e1\",\n \"name\": \"web-origins\",\n \"description\": \"OpenID Connect scope for add allowed web origins to the access token\",\n \"protocol\": \"openid-connect\",\n \"attributes\": {\n \"include.in.token.scope\": \"false\",\n \"consent.screen.text\": \"\",\n \"display.on.consent.screen\": \"false\"\n },\n \"protocolMappers\": [\n {\n \"id\": \"2038278c-eb80-42eb-828a-ce1a700bcf5d\",\n \"name\": \"allowed web origins\",\n \"protocol\": \"openid-connect\",\n \"protocolMapper\": \"oidc-allowed-origins-mapper\",\n \"consentRequired\": false,\n \"config\": {\n \"introspection.token.claim\": \"true\",\n \"access.token.claim\": \"true\"\n }\n }\n ]\n }\n ],\n \"defaultDefaultClientScopes\": [\n \"role_list\",\n \"saml_organization\",\n \"profile\",\n \"email\",\n \"roles\",\n \"web-origins\",\n \"acr\",\n \"basic\"\n ],\n \"defaultOptionalClientScopes\": [\n \"offline_access\",\n \"address\",\n \"phone\",\n \"microprofile-jwt\",\n \"organization\"\n ],\n \"browserSecurityHeaders\": {\n \"contentSecurityPolicyReportOnly\": \"\",\n \"xContentTypeOptions\": \"nosniff\",\n \"referrerPolicy\": \"no-referrer\",\n \"xRobotsTag\": \"none\",\n \"xFrameOptions\": \"SAMEORIGIN\",\n \"contentSecurityPolicy\": \"frame-src 'self'; frame-ancestors 'self'; object-src 'none';\",\n \"strictTransportSecurity\": \"max-age=31536000; includeSubDomains\"\n },\n \"smtpServer\": {},\n \"eventsEnabled\": false,\n \"eventsListeners\": [\n \"jboss-logging\"\n ],\n \"enabledEventTypes\": [],\n \"adminEventsEnabled\": false,\n \"adminEventsDetailsEnabled\": false,\n \"identityProviders\": [],\n \"identityProviderMappers\": [],\n \"components\": {\n \"org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy\": [\n {\n \"id\": \"816e3705-90b1-4aba-988e-100ff1effac8\",\n \"name\": \"Allowed Client Scopes\",\n \"providerId\": \"allowed-client-templates\",\n \"subType\": \"authenticated\",\n \"subComponents\": {},\n \"config\": {\n \"allow-default-scopes\": [\n \"true\"\n ]\n }\n },\n {\n \"id\": \"d17f76e9-3fc9-4e0b-ba62-1787cfaacf21\",\n \"name\": \"Trusted Hosts\",\n \"providerId\": \"trusted-hosts\",\n \"subType\": \"anonymous\",\n \"subComponents\": {},\n \"config\": {\n \"host-sending-registration-request-must-match\": [\n \"true\"\n ],\n \"client-uris-must-match\": [\n \"true\"\n ]\n }\n },\n {\n \"id\": \"de42e33e-cf5f-40c6-8c0d-7bb4fd7cb91b\",\n \"name\": \"Allowed Client Scopes\",\n \"providerId\": \"allowed-client-templates\",\n \"subType\": \"anonymous\",\n \"subComponents\": {},\n \"config\": {\n \"allow-default-scopes\": [\n \"true\"\n ]\n }\n },\n {\n \"id\": \"a5989451-aad7-4b75-85cf-29340c3508d6\",\n \"name\": \"Allowed Protocol Mapper Types\",\n \"providerId\": \"allowed-protocol-mappers\",\n \"subType\": \"authenticated\",\n \"subComponents\": {},\n \"config\": {\n \"allowed-protocol-mapper-types\": [\n \"saml-user-property-mapper\",\n \"oidc-full-name-mapper\",\n \"oidc-address-mapper\",\n \"oidc-usermodel-property-mapper\",\n \"oidc-usermodel-attribute-mapper\",\n \"saml-user-attribute-mapper\",\n \"saml-role-list-mapper\",\n \"oidc-sha256-pairwise-sub-mapper\"\n ]\n }\n },\n {\n \"id\": \"871c280c-5f17-4ec4-9fd6-f36171d56553\",\n \"name\": \"Max Clients Limit\",\n \"providerId\": \"max-clients\",\n \"subType\": \"anonymous\",\n \"subComponents\": {},\n \"config\": {\n \"max-clients\": [\n \"200\"\n ]\n }\n },\n {\n \"id\": \"fbf6f1c2-aabf-40ef-b764-8a677603d2cc\",\n \"name\": \"Consent Required\",\n \"providerId\": \"consent-required\",\n \"subType\": \"anonymous\",\n \"subComponents\": {},\n \"config\": {}\n },\n {\n \"id\": \"a6dc5c32-28cd-424f-b8ec-db381d5a8fdd\",\n \"name\": \"Allowed Protocol Mapper Types\",\n \"providerId\": \"allowed-protocol-mappers\",\n \"subType\": \"anonymous\",\n \"subComponents\": {},\n \"config\": {\n \"allowed-protocol-mapper-types\": [\n \"oidc-address-mapper\",\n \"saml-user-attribute-mapper\",\n \"oidc-usermodel-attribute-mapper\",\n \"oidc-full-name-mapper\",\n \"saml-user-property-mapper\",\n \"oidc-sha256-pairwise-sub-mapper\",\n \"oidc-usermodel-property-mapper\",\n \"saml-role-list-mapper\"\n ]\n }\n },\n {\n \"id\": \"798a0205-2a20-436d-b15f-ec19ee70a277\",\n \"name\": \"Full Scope Disabled\",\n \"providerId\": \"scope\",\n \"subType\": \"anonymous\",\n \"subComponents\": {},\n \"config\": {}\n }\n ],\n \"org.keycloak.keys.KeyProvider\": [\n {\n \"id\": \"702a660b-9d47-4d4e-b0eb-326775910293\",\n \"name\": \"aes-generated\",\n \"providerId\": \"aes-generated\",\n \"subComponents\": {},\n \"config\": {\n \"priority\": [\n \"100\"\n ]\n }\n },\n {\n \"id\": \"025d31ab-f248-4358-87d0-ec0e27bee344\",\n \"name\": \"rsa-generated\",\n \"providerId\": \"rsa-generated\",\n \"subComponents\": {},\n \"config\": {\n \"priority\": [\n \"100\"\n ]\n }\n },\n {\n \"id\": \"efec8cce-6c75-4e89-9017-f553468e2811\",\n \"name\": \"rsa-enc-generated\",\n \"providerId\": \"rsa-enc-generated\",\n \"subComponents\": {},\n \"config\": {\n \"priority\": [\n \"100\"\n ],\n \"algorithm\": [\n \"RSA-OAEP\"\n ]\n }\n },\n {\n \"id\": \"40d62ca1-93bc-4110-b2dc-7d7079c45774\",\n \"name\": \"hmac-generated-hs512\",\n \"providerId\": \"hmac-generated\",\n \"subComponents\": {},\n \"config\": {\n \"priority\": [\n \"100\"\n ],\n \"algorithm\": [\n \"HS512\"\n ]\n }\n }\n ]\n },\n \"internationalizationEnabled\": false,\n \"supportedLocales\": [],\n \"authenticationFlows\": [\n {\n \"id\": \"3ce2ce6d-7743-4e1b-8a31-291fcc7ae06f\",\n \"alias\": \"Account verification options\",\n \"description\": \"Method with which to verity the existing account\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"idp-email-verification\",\n \"authenticatorFlow\": false,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 20,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"Verify Existing Account by Re-authentication\",\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"8267719e-7856-4694-a9eb-f405eca2aca5\",\n \"alias\": \"Browser - Conditional OTP\",\n \"description\": \"Flow to determine if the OTP is required for the authentication\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"conditional-user-configured\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"auth-otp-form\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 20,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"596aae9f-ae33-415a-b9e8-de70ee954ac5\",\n \"alias\": \"Browser - Conditional Organization\",\n \"description\": \"Flow to determine if the organization identity-first login is to be used\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"conditional-user-configured\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"organization\",\n \"authenticatorFlow\": false,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 20,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"068702f9-b6d9-489e-af34-8b9f325a361b\",\n \"alias\": \"Direct Grant - Conditional OTP\",\n \"description\": \"Flow to determine if the OTP is required for the authentication\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"conditional-user-configured\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"direct-grant-validate-otp\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 20,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"0459a690-8543-422c-bc83-94e8c62c8bdd\",\n \"alias\": \"First Broker Login - Conditional Organization\",\n \"description\": \"Flow to determine if the authenticator that adds organization members is to be used\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"conditional-user-configured\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"idp-add-organization-member\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 20,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"31cb5a9b-b319-45d7-980d-cc6cd8a9e579\",\n \"alias\": \"First broker login - Conditional OTP\",\n \"description\": \"Flow to determine if the OTP is required for the authentication\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"conditional-user-configured\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"auth-otp-form\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 20,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"7448795c-59cf-4a43-9af8-a988cdc1e7b6\",\n \"alias\": \"Handle Existing Account\",\n \"description\": \"Handle what to do if there is existing account with same email/username like authenticated identity provider\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"idp-confirm-link\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"REQUIRED\",\n \"priority\": 20,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"Account verification options\",\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"c18be560-2806-4c78-942d-00d4e32b0339\",\n \"alias\": \"Organization\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"CONDITIONAL\",\n \"priority\": 10,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"Browser - Conditional Organization\",\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"498a92fe-00bb-4419-a9e1-c08dcc55c0b9\",\n \"alias\": \"Reset - Conditional OTP\",\n \"description\": \"Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"conditional-user-configured\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"reset-otp\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 20,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"f19dc4c2-1167-41f7-8573-ecf19f285c9e\",\n \"alias\": \"User creation or linking\",\n \"description\": \"Flow for the existing/non-existing user alternatives\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticatorConfig\": \"create unique user config\",\n \"authenticator\": \"idp-create-user-if-unique\",\n \"authenticatorFlow\": false,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 20,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"Handle Existing Account\",\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"fb78126b-7892-4da6-adc0-58a9b74e0a0d\",\n \"alias\": \"Verify Existing Account by Re-authentication\",\n \"description\": \"Reauthentication of existing account\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"idp-username-password-form\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"CONDITIONAL\",\n \"priority\": 20,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"First broker login - Conditional OTP\",\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"2a393886-1bea-48bd-823b-485d4ae4a6a6\",\n \"alias\": \"browser\",\n \"description\": \"Browser based authentication\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": true,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"auth-cookie\",\n \"authenticatorFlow\": false,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"auth-spnego\",\n \"authenticatorFlow\": false,\n \"requirement\": \"DISABLED\",\n \"priority\": 20,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"identity-provider-redirector\",\n \"authenticatorFlow\": false,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 25,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 26,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"Organization\",\n \"userSetupAllowed\": false\n },\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 30,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"forms\",\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"8df87113-974a-43b0-80cc-4a86a8eb907a\",\n \"alias\": \"clients\",\n \"description\": \"Base authentication for clients\",\n \"providerId\": \"client-flow\",\n \"topLevel\": true,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"client-secret\",\n \"authenticatorFlow\": false,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"client-jwt\",\n \"authenticatorFlow\": false,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 20,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"client-secret-jwt\",\n \"authenticatorFlow\": false,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 30,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"client-x509\",\n \"authenticatorFlow\": false,\n \"requirement\": \"ALTERNATIVE\",\n \"priority\": 40,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"f3116f33-e9e1-439d-8997-5496b244640e\",\n \"alias\": \"direct grant\",\n \"description\": \"OpenID Connect Resource Owner Grant\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": true,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"direct-grant-validate-username\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"direct-grant-validate-password\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 20,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"CONDITIONAL\",\n \"priority\": 30,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"Direct Grant - Conditional OTP\",\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"05f713a9-caaf-4f5f-8df0-3423ee1d3b2e\",\n \"alias\": \"docker auth\",\n \"description\": \"Used by Docker clients to authenticate against the IDP\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": true,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"docker-http-basic-authenticator\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"caa3d932-2aad-42c4-9e63-de8d32c8858c\",\n \"alias\": \"first broker login\",\n \"description\": \"Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": true,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticatorConfig\": \"review profile config\",\n \"authenticator\": \"idp-review-profile\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"REQUIRED\",\n \"priority\": 20,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"User creation or linking\",\n \"userSetupAllowed\": false\n },\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"CONDITIONAL\",\n \"priority\": 50,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"First Broker Login - Conditional Organization\",\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"eba73d10-65cf-4364-8969-92571fe69b72\",\n \"alias\": \"forms\",\n \"description\": \"Username, password, otp and other auth forms.\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"auth-username-password-form\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"CONDITIONAL\",\n \"priority\": 20,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"Browser - Conditional OTP\",\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"e2d23ce6-e37d-46f1-83fd-b18076d81aeb\",\n \"alias\": \"registration\",\n \"description\": \"Registration flow\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": true,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"registration-page-form\",\n \"authenticatorFlow\": true,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"registration form\",\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"00d69e38-630a-4215-a635-e71849ae2abc\",\n \"alias\": \"registration form\",\n \"description\": \"Registration form\",\n \"providerId\": \"form-flow\",\n \"topLevel\": false,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"registration-user-creation\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 20,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"registration-password-action\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 50,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"registration-recaptcha-action\",\n \"authenticatorFlow\": false,\n \"requirement\": \"DISABLED\",\n \"priority\": 60,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"registration-terms-and-conditions\",\n \"authenticatorFlow\": false,\n \"requirement\": \"DISABLED\",\n \"priority\": 70,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"fe3aae15-d563-4576-a709-4ba3a40c540e\",\n \"alias\": \"reset credentials\",\n \"description\": \"Reset credentials for a user if they forgot their password or something\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": true,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"reset-credentials-choose-user\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"reset-credential-email\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 20,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticator\": \"reset-password\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 30,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n },\n {\n \"authenticatorFlow\": true,\n \"requirement\": \"CONDITIONAL\",\n \"priority\": 40,\n \"autheticatorFlow\": true,\n \"flowAlias\": \"Reset - Conditional OTP\",\n \"userSetupAllowed\": false\n }\n ]\n },\n {\n \"id\": \"ffedec01-f274-4b58-ab70-178ff4e8c569\",\n \"alias\": \"saml ecp\",\n \"description\": \"SAML ECP Profile Authentication Flow\",\n \"providerId\": \"basic-flow\",\n \"topLevel\": true,\n \"builtIn\": true,\n \"authenticationExecutions\": [\n {\n \"authenticator\": \"http-basic-authenticator\",\n \"authenticatorFlow\": false,\n \"requirement\": \"REQUIRED\",\n \"priority\": 10,\n \"autheticatorFlow\": false,\n \"userSetupAllowed\": false\n }\n ]\n }\n ],\n \"authenticatorConfig\": [\n {\n \"id\": \"3911f29c-6c6a-418c-93a8-72ba806d715b\",\n \"alias\": \"create unique user config\",\n \"config\": {\n \"require.password.update.after.registration\": \"false\"\n }\n },\n {\n \"id\": \"dc544a68-9478-4679-8a61-76e9afb84757\",\n \"alias\": \"review profile config\",\n \"config\": {\n \"update.profile.on.first.login\": \"missing\"\n }\n }\n ],\n \"requiredActions\": [\n {\n \"alias\": \"CONFIGURE_TOTP\",\n \"name\": \"Configure OTP\",\n \"providerId\": \"CONFIGURE_TOTP\",\n \"enabled\": true,\n \"defaultAction\": false,\n \"priority\": 10,\n \"config\": {}\n },\n {\n \"alias\": \"TERMS_AND_CONDITIONS\",\n \"name\": \"Terms and Conditions\",\n \"providerId\": \"TERMS_AND_CONDITIONS\",\n \"enabled\": false,\n \"defaultAction\": false,\n \"priority\": 20,\n \"config\": {}\n },\n {\n \"alias\": \"UPDATE_PASSWORD\",\n \"name\": \"Update Password\",\n \"providerId\": \"UPDATE_PASSWORD\",\n \"enabled\": true,\n \"defaultAction\": false,\n \"priority\": 30,\n \"config\": {}\n },\n {\n \"alias\": \"UPDATE_PROFILE\",\n \"name\": \"Update Profile\",\n \"providerId\": \"UPDATE_PROFILE\",\n \"enabled\": true,\n \"defaultAction\": false,\n \"priority\": 40,\n \"config\": {}\n },\n {\n \"alias\": \"VERIFY_EMAIL\",\n \"name\": \"Verify Email\",\n \"providerId\": \"VERIFY_EMAIL\",\n \"enabled\": true,\n \"defaultAction\": false,\n \"priority\": 50,\n \"config\": {}\n },\n {\n \"alias\": \"delete_account\",\n \"name\": \"Delete Account\",\n \"providerId\": \"delete_account\",\n \"enabled\": false,\n \"defaultAction\": false,\n \"priority\": 60,\n \"config\": {}\n },\n {\n \"alias\": \"webauthn-register\",\n \"name\": \"Webauthn Register\",\n \"providerId\": \"webauthn-register\",\n \"enabled\": true,\n \"defaultAction\": false,\n \"priority\": 70,\n \"config\": {}\n },\n {\n \"alias\": \"webauthn-register-passwordless\",\n \"name\": \"Webauthn Register Passwordless\",\n \"providerId\": \"webauthn-register-passwordless\",\n \"enabled\": true,\n \"defaultAction\": false,\n \"priority\": 80,\n \"config\": {}\n },\n {\n \"alias\": \"VERIFY_PROFILE\",\n \"name\": \"Verify Profile\",\n \"providerId\": \"VERIFY_PROFILE\",\n \"enabled\": true,\n \"defaultAction\": false,\n \"priority\": 90,\n \"config\": {}\n },\n {\n \"alias\": \"delete_credential\",\n \"name\": \"Delete Credential\",\n \"providerId\": \"delete_credential\",\n \"enabled\": true,\n \"defaultAction\": false,\n \"priority\": 100,\n \"config\": {}\n },\n {\n \"alias\": \"update_user_locale\",\n \"name\": \"Update User Locale\",\n \"providerId\": \"update_user_locale\",\n \"enabled\": true,\n \"defaultAction\": false,\n \"priority\": 1000,\n \"config\": {}\n }\n ],\n \"browserFlow\": \"browser\",\n \"registrationFlow\": \"registration\",\n \"directGrantFlow\": \"direct grant\",\n \"resetCredentialsFlow\": \"reset credentials\",\n \"clientAuthenticationFlow\": \"clients\",\n \"dockerAuthenticationFlow\": \"docker auth\",\n \"firstBrokerLoginFlow\": \"first broker login\",\n \"attributes\": {\n \"cibaBackchannelTokenDeliveryMode\": \"poll\",\n \"cibaExpiresIn\": \"120\",\n \"cibaAuthRequestedUserHint\": \"login_hint\",\n \"oauth2DeviceCodeLifespan\": \"600\",\n \"clientOfflineSessionMaxLifespan\": \"0\",\n \"oauth2DevicePollingInterval\": \"5\",\n \"clientSessionIdleTimeout\": \"0\",\n \"parRequestUriLifespan\": \"60\",\n \"clientSessionMaxLifespan\": \"0\",\n \"clientOfflineSessionIdleTimeout\": \"0\",\n \"cibaInterval\": \"5\",\n \"realmReusableOtpCode\": \"false\"\n },\n \"keycloakVersion\": \"26.2.5\",\n \"userManagedAccessAllowed\": false,\n \"organizationsEnabled\": false,\n \"verifiableCredentialsEnabled\": false,\n \"adminPermissionsEnabled\": false,\n \"clientProfiles\": {\n \"profiles\": []\n },\n \"clientPolicies\": {\n \"policies\": []\n }\n}" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DataSourceAggregationTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.junit.Assert.assertEquals\nimport org.junit.Test\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.summary.ResultSummary\n\nclass DataSourceReaderAggregationTSE extends SparkConnectorScalaBaseTSE {\n\n @Test\n def testShouldDoSumAggregation(): Unit = {\n val fixtureQuery: String =\n s\"\"\"CREATE (pe:Person {id: 1, fullName: 'Person'})-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr:Product {id: 0, name: 'Product ' + 0, price: 1})\n |WITH pe\n |UNWIND range(1, 10) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id, price: id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Product\")\n .load\n .createTempView(\"BOUGHT\")\n\n val df = ss.sql(\n \"\"\"SELECT `source.fullName`, SUM(DISTINCT(`target.price`)) AS distinctTotal, SUM(`target.price`) AS total\n |FROM BOUGHT\n |group by `source.fullName`\"\"\".stripMargin\n )\n\n val rows = df.collect().toList\n assertEquals(1, rows.length)\n val row = rows(0)\n assertEquals(\"Person\", row.getAs[String](\"source.fullName\"))\n assertEquals(55L, row.getAs[Long](\"distinctTotal\"))\n assertEquals(56L, row.getAs[Long](\"total\"))\n }\n\n @Test\n def testShouldDoMaxMinAggregation(): Unit = {\n val fixtureQuery: String =\n s\"\"\"CREATE (pe:Person {id: 1, fullName: 'Person'})\n |WITH pe\n |UNWIND range(1, 10) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id, price: id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Product\")\n .load\n .createTempView(\"BOUGHT\")\n\n val df = ss.sql(\n \"\"\"SELECT `source.fullName`, MAX(`target.price`) AS max, MIN(`target.price`) AS min\n |FROM BOUGHT\n |GROUP BY `source.fullName`\"\"\".stripMargin\n )\n\n val rows = df.collect().toList\n assertEquals(1, rows.length)\n val row = rows(0)\n assertEquals(\"Person\", row.getAs[String](\"source.fullName\"))\n assertEquals(10L, row.getAs[Long](\"max\"))\n assertEquals(1L, row.getAs[Long](\"min\"))\n }\n\n @Test\n def testShouldDoCountAggregation(): Unit = {\n val fixtureQuery: String =\n s\"\"\"CREATE (pe:Person {id: 1, fullName: 'Person'})-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr:Product {id: 1, name: 'Product 1', price: 1})\n |WITH pe\n |UNWIND range(1, 10) as id\n |MERGE (pr:Product {id: id, name: 'Product ' + id, price: id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Product\")\n .load\n .createTempView(\"BOUGHT\")\n\n val df = ss.sql(\n \"\"\"SELECT `source.fullName`, COUNT(DISTINCT(`target.id`)) AS distinctTotal, COUNT(`target.id`) AS total\n |FROM BOUGHT\n |group by `source.fullName`\"\"\".stripMargin\n )\n\n val rows = df.collect().toList\n assertEquals(1, rows.length)\n val row = rows(0)\n assertEquals(\"Person\", row.getAs[String](\"source.fullName\"))\n assertEquals(10L, row.getAs[Long](\"distinctTotal\"))\n assertEquals(11L, row.getAs[Long](\"total\"))\n\n }\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DataSourceReaderNeo4jTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.SparkException\nimport org.apache.spark.sql.DataFrame\nimport org.junit.Assert.assertEquals\nimport org.junit.Assert.assertTrue\nimport org.junit.Assert.fail\nimport org.junit.Assume\nimport org.junit.BeforeClass\nimport org.junit.Test\nimport org.neo4j.Closeables.use\nimport org.neo4j.driver.SessionConfig\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.exceptions.ClientException\nimport org.neo4j.driver.summary.ResultSummary\n\nclass DataSourceReaderNeo4jTSE extends SparkConnectorScalaBaseTSE {\n\n @Test\n def testMultiDbJoin(): Unit = {\n SparkConnectorScalaSuiteIT.driver.session(SessionConfig.forDatabase(\"db1\"))\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(\n \"\"\"\n CREATE (p1:Person:Customer {name: 'John Doe'}),\n (p2:Person:Customer {name: 'Mark Brown'}),\n (p3:Person:Customer {name: 'Cindy White'})\n \"\"\"\n ).consume()\n }\n )\n\n SparkConnectorScalaSuiteIT.driver.session(SessionConfig.forDatabase(\"db2\"))\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(\n \"\"\"\n CREATE (p1:Person:Employee {name: 'Jane Doe'}),\n (p2:Person:Employee {name: 'John Doe'})\n \"\"\"\n ).consume()\n }\n )\n\n val df1 = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"labels\", \"Person\")\n .load()\n\n val df2 = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"labels\", \"Person\")\n .load()\n\n assertEquals(3, df1.count())\n assertEquals(2, df2.count())\n\n val dfJoin = df1.join(df2, df1(\"name\") === df2(\"name\"))\n assertEquals(1, dfJoin.count())\n }\n\n @Test\n def testReadQueryCustomPartitions(): Unit = {\n val fixtureProduct1Query: String =\n \"\"\"CREATE (pr:Product{id: 1, name: 'Product 1'})\n |WITH pr\n |UNWIND range(1,100) as id\n |CREATE (p:Person {id: id, name: 'Person ' + id})-[:BOUGHT{quantity: ceil(rand() * 100)}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n SparkConnectorScalaSuiteIT.driver.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureProduct1Query).consume()\n }\n )\n val fixtureProduct2Query: String =\n \"\"\"CREATE (pr:Product{id: 2, name: 'Product 2'})\n |WITH pr\n |UNWIND range(1,50) as id\n |MATCH (p:Person {id: id})\n |CREATE (p)-[:BOUGHT{quantity: ceil(rand() * 100)}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n SparkConnectorScalaSuiteIT.driver.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureProduct2Query).consume()\n }\n )\n\n val partitionedDf = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"\n |MATCH (p:Person)-[r:BOUGHT]->(pr:Product)\n |RETURN p.name AS person, pr.name AS product, r.quantity AS quantity\"\"\".stripMargin\n )\n .option(\"partitions\", \"5\")\n .load()\n\n assertEquals(5, partitionedDf.rdd.getNumPartitions)\n val rows = partitionedDf.collect()\n .map(row => s\"${row.getAs[String](\"person\")}-${row.getAs[String](\"product\")}\")\n assertEquals(150, rows.size)\n assertEquals(150, rows.size)\n }\n\n @Test\n def testCallShouldReturnCorrectSchema(): Unit = {\n val callDf: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"CALL db.info() YIELD id, name RETURN *\")\n .load()\n\n val res = callDf.select(\"name\")\n .collectAsList()\n .get(0)\n\n assertEquals(res.getString(0), \"neo4j\")\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithNode(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id, name: 'Product ' + id})\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Product\")\n .load\n .select(\"name\")\n\n df.count()\n\n assertEquals(Seq(\"name\"), df.columns.toSeq)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithNodeAndWeirdColumnName(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id, `(╯°□°)╯︵ ┻━┻`: 'Product ' + id})\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Product\")\n .load\n .select(\"`(╯°□°)╯︵ ┻━┻`\")\n\n df.count()\n\n assertEquals(Seq(\"(╯°□°)╯︵ ┻━┻\"), df.columns.toSeq)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithRelationship(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Product\")\n .option(\"relationship.target.labels\", \"Person\")\n .load\n .select(\"`source.name`\", \"``\")\n\n df.count()\n\n assertEquals(Seq(\"source.name\", \"\"), df.columns.toSeq)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithRelationshipAndWeirdColumn(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), `(╯°□°)╯︵ ┻━┻`: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Product\")\n .load\n .select(\"`target.(╯°□°)╯︵ ┻━┻`\", \"``\")\n\n df.count()\n\n assertEquals(Seq(\"target.(╯°□°)╯︵ ┻━┻\", \"\"), df.columns.toSeq)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithQuery(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (p:Product) RETURN p.name as name\")\n .option(\"partitions\", 2)\n .option(\"query.count\", 20)\n .load\n .select(\"name\")\n\n df.count()\n\n assertEquals(Seq(\"name\"), df.columns.toSeq)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithFilter(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id, name: 'Product ' + id})\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Product\")\n .load\n .filter(\"name = 'Product 1'\")\n\n df.count()\n\n assertEquals(Seq(\"\", \"\", \"name\", \"id\"), df.columns.toSeq)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithRelationshipWithFilter(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Product\")\n .load\n .filter(\"`target.name` = 'Product 1' AND `target.id` = '16'\")\n .select(\"`target.name`\", \"`target.id`\")\n\n df.count()\n\n assertEquals(Seq(\"target.name\", \"target.id\"), df.columns.toSeq)\n }\n\n @Test\n def testShouldThrowClearErrorIfAWrongDbIsSpecified(): Unit = {\n try {\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"not_existing_db\")\n .option(\"labels\", \"MATCH (h:Household) RETURN id(h)\")\n .load()\n .show()\n } catch {\n case clientException: ClientException => {\n assertTrue(clientException.getMessage.equals(\n \"Database does not exist. Database name: 'not_existing_db'.\"\n ))\n }\n case generic: Throwable =>\n fail(s\"should be thrown a ${classOf[SparkException].getName}, got ${generic.getClass} instead\")\n }\n }\n\n @Test\n def testEmptyDataset(): Unit = {\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (e:ID_DO_NOT_EXIST) RETURN id(e) as f, 1 as g\")\n .load\n\n assertEquals(0, df.count())\n assertEquals(Seq(\"f\", \"g\"), df.columns.toSeq)\n }\n\n @Test\n def testColumnSorted(): Unit = {\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary =\n tx.run(\"CREATE (i1:Instrument{name: 'Drums', id: 1}), (i2:Instrument{name: 'Guitar', id: 2})\").consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (i:Instrument) RETURN id(i) as internal_id, i.id as id, i.name as name, i.name\")\n .load\n .orderBy(\"id\")\n\n assertEquals(1L, df.collectAsList().get(0).get(1))\n assertEquals(\"Drums\", df.collectAsList().get(0).get(2))\n assertEquals(Seq(\"internal_id\", \"id\", \"name\", \"i.name\"), df.columns.toSeq)\n }\n\n @Test\n def testComplexReturnStatement(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n use(SparkConnectorScalaSuiteIT.session()) { session =>\n session\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n }\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"MATCH (p:Person)-[b:BOUGHT]->(pr:Product)\n |RETURN id(p) AS personId, id(pr) AS productId, {quantity: b.quantity, when: b.when} AS map, \"some string\" as someString, {anotherField: \"201\"} as map2\"\"\".stripMargin\n )\n .option(\"schema.strategy\", \"string\")\n .load()\n\n assertEquals(Seq(\"personId\", \"productId\", \"map\", \"someString\", \"map2\"), df.columns.toSeq)\n assertEquals(100, df.count())\n }\n\n @Test\n def testComplexReturnStatementNoValues(): Unit = {\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"MATCH (p:Person)-[b:BOUGHT]->(pr:Product)\n |RETURN id(p) AS personId, id(pr) AS productId, {quantity: b.quantity, when: b.when} AS map, \"some string\" as someString, {anotherField: \"201\", and: 1} as map2\"\"\".stripMargin\n )\n .option(\"schema.strategy\", \"string\")\n .load()\n\n assertEquals(Seq(\"personId\", \"productId\", \"map\", \"someString\", \"map2\"), df.columns.toSeq)\n assertEquals(0, df.count())\n }\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DataSourceReaderNeo4jWithApocTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.junit.Assert.assertEquals\nimport org.junit.Assume\nimport org.junit.BeforeClass\nimport org.junit.Test\nimport org.neo4j.driver.SessionConfig\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.summary.ResultSummary\n\nclass DataSourceReaderNeo4jWithApocTSE extends SparkConnectorScalaBaseWithApocTSE {\n\n @Test\n def testMultiDbJoin(): Unit = {\n SparkConnectorScalaSuiteWithApocIT.driver.session(SessionConfig.forDatabase(\"db1\"))\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(\n \"\"\"\n CREATE (p1:Person:Customer {name: 'John Doe'}),\n (p2:Person:Customer {name: 'Mark Brown'}),\n (p3:Person:Customer {name: 'Cindy White'})\n \"\"\"\n ).consume()\n }\n )\n\n SparkConnectorScalaSuiteWithApocIT.driver.session(SessionConfig.forDatabase(\"db2\"))\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(\n \"\"\"\n CREATE (p1:Person:Employee {name: 'Jane Doe'}),\n (p2:Person:Employee {name: 'John Doe'})\n \"\"\"\n ).consume()\n }\n )\n\n val df1 = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"labels\", \"Person\")\n .load()\n\n val df2 = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"labels\", \"Person\")\n .load()\n\n assertEquals(3, df1.count())\n assertEquals(2, df2.count())\n\n val dfJoin = df1.join(df2, df1(\"name\") === df2(\"name\"))\n assertEquals(1, dfJoin.count())\n }\n\n @Test\n def testReturnProcedure(): Unit = {\n val query =\n \"\"\"RETURN apoc.convert.toSet([1,1,3]) AS foo, 'bar' AS bar\n |\"\"\".stripMargin\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"partitions\", 1)\n .option(\"query\", query)\n .load\n\n assertEquals(Seq(\"foo\", \"bar\"), df.columns.toSeq) // ordering should be preserved\n assertEquals(1, df.count())\n }\n\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DataSourceReaderTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.sql.DataFrame\nimport org.apache.spark.sql.Row\nimport org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema\nimport org.apache.spark.sql.functions.col\nimport org.apache.spark.sql.types.DataTypes\nimport org.apache.spark.sql.types.StructField\nimport org.apache.spark.sql.types.StructType\nimport org.junit.Assert._\nimport org.junit.Test\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.summary.ResultSummary\n\nimport java.sql.Timestamp\nimport java.time._\nimport java.util.TimeZone\n\nimport scala.collection.JavaConverters._\nimport scala.collection.mutable.Seq\n\nclass DataSourceReaderTSE extends SparkConnectorScalaBaseTSE {\n\n @Test\n def testThrowsExceptionIfNoValidReadOptionIsSet(): Unit = {\n try {\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .load()\n .show() // we need the action to be able to trigger the exception because of the changes in Spark 3\n org.junit.Assert.fail(\"Expected to throw an exception\")\n } catch {\n case e: IllegalArgumentException =>\n assertEquals(\"No valid option found. One of `GDS`, `LABELS`, `QUERY`, `RELATIONSHIP` is required\", e.getMessage)\n case _: Throwable => fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n\n @Test\n def testThrowsExceptionIfTwoValidReadOptionAreSet(): Unit = {\n try {\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .option(\"relationship\", \"KNOWS\")\n .load()\n .show() // we need the action to be able to trigger the exception because of the changes in Spark 3\n org.junit.Assert.fail(\"Expected to throw an exception\")\n } catch {\n case e: IllegalArgumentException =>\n assertEquals(\n \"You need to specify just one of these options: 'gds', 'labels', 'query', 'relationship'\",\n e.getMessage\n )\n case _: Throwable => fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n\n @Test\n def testThrowsExceptionIfThreeValidReadOptionAreSet(): Unit = {\n try {\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .option(\"relationship\", \"KNOWS\")\n .option(\"query\", \"MATCH (n) RETURN n\")\n .load()\n .show() // we need the action to be able to trigger the exception because of the changes in Spark 3\n org.junit.Assert.fail(\"Expected to throw an exception\")\n } catch {\n case e: IllegalArgumentException =>\n assertEquals(\n \"You need to specify just one of these options: 'gds', 'labels', 'query', 'relationship'\",\n e.getMessage\n )\n case _: Throwable => fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n\n @Test\n def testReadNodeHasIdField(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {name: 'John'})\")\n\n /**\n * utnaf: Since we can't be sure we are in total isolation, and the id is generated\n * internally by org.neo4j.neo4j, we just check that the field is an integer and is greater\n * than -1\n */\n assertTrue(df.select(\"\").collectAsList().get(0).getLong(0) > -1)\n }\n\n @Test\n def testReadNodeHasLabelsField(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person:Customer {name: 'John'})\")\n\n val result = df.select(\"\").collectAsList().get(0).getAs[Seq[String]](0)\n\n assertEquals(\"Person\", result.head)\n assertEquals(\"Customer\", result(1))\n }\n\n @Test\n def testReadNodeHasUnusualLabelsField(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:`Foo Bar`:Person:`(╯°□°）╯︵ ┻━┻` {name: 'John'})\")\n\n val result = df.select(\"\").collectAsList().get(0).getAs[Seq[String]](0)\n\n assertEquals(Set(\"Person\", \"Foo Bar\", \"(╯°□°）╯︵ ┻━┻\"), result.toSet[String])\n }\n\n @Test\n def testReadNodeWithFieldWithDifferentTypes(): Unit = {\n val df: DataFrame = initTest(\"CREATE (p1:Person {id: 1, field: [12,34]}), (p2:Person {id: 2, field: 123})\")\n\n val res = df.orderBy(\"id\").collectAsList()\n\n assertEquals(\"[12,34]\", res.get(0).get(3))\n assertEquals(\"123\", res.get(1).get(3))\n }\n\n @Test\n def testReadNodeWithString(): Unit = {\n val name: String = \"John\"\n val df: DataFrame = initTest(s\"CREATE (p:Person {name: '$name'})\")\n\n assertEquals(name, df.select(\"name\").collectAsList().get(0).getString(0))\n }\n\n @Test\n def testReadNodeWithLong(): Unit = {\n val age: Long = 42\n val df: DataFrame = initTest(s\"CREATE (p:Person {age: $age})\")\n\n assertEquals(age, df.select(\"age\").collectAsList().get(0).getLong(0))\n }\n\n @Test\n def testReadNodeWithDouble(): Unit = {\n val score: Double = 3.14\n val df: DataFrame = initTest(s\"CREATE (p:Person {score: $score})\")\n\n assertEquals(score, df.select(\"score\").collectAsList().get(0).getDouble(0), 0)\n }\n\n @Test\n def testReadNodeWithLocalTime(): Unit = {\n val df: DataFrame =\n initTest(s\"CREATE (p:Person {aTime: localtime({hour:12, minute: 23, second: 0, millisecond: 294})})\")\n\n val result = df.select(\"aTime\").collectAsList().get(0).getAs[GenericRowWithSchema](0)\n assertEquals(\"local-time\", result.get(0))\n assertEquals(\"12:23:00.294\", result.get(1))\n }\n\n @Test\n def testReadNodeWithTime(): Unit = {\n val timezone = TimeZone.getDefault\n val df: DataFrame = initTest(s\"CREATE (p:Person {aTime: time({hour:12, minute: 23, second: 0, millisecond: 294})})\")\n\n val result = df.select(\"aTime\").collectAsList().get(0).getAs[GenericRowWithSchema](0)\n\n val localTime = LocalTime.of(12, 23, 0, 294000000)\n val expectedTime = OffsetTime.of(localTime, timezone.toZoneId.getRules.getOffset(Instant.now))\n\n assertEquals(\"offset-time\", result.get(0))\n assertEquals(expectedTime.toString, result.get(1))\n }\n\n @Test\n def testReadNodeWithLocalDateTime(): Unit = {\n val localDateTime = \"2007-12-03T10:15:30\"\n val df: DataFrame = initTest(s\"CREATE (p:Person {aTime: localdatetime('$localDateTime')})\")\n val result = df.select(\"aTime\").collectAsList().get(0).getAs[LocalDateTime](0)\n assertEquals(LocalDateTime.parse(localDateTime), result)\n }\n\n @Test\n def testReadNodeWithZonedDateTime(): Unit = {\n val datetime = \"2015-06-24T12:50:35.556+01:00\"\n val df: DataFrame = initTest(s\"CREATE (p:Person {aTime: datetime('$datetime')})\")\n\n val result = df.select(\"aTime\").collectAsList().get(0).getTimestamp(0)\n\n assertEquals(Timestamp.from(OffsetDateTime.parse(datetime).toInstant), result)\n }\n\n @Test\n def testReadNodeWithPoint(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {location: point({x: 12.12, y: 13.13})})\")\n\n val res = df.select(\"location\").collectAsList().get(0).getAs[GenericRowWithSchema](0);\n\n assertEquals(\"point-2d\", res.get(0))\n assertEquals(7203, res.get(1))\n assertEquals(12.12, res.get(2))\n assertEquals(13.13, res.get(3))\n }\n\n @Test\n def testReadNodeWithGeoPoint(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {location: point({longitude: 12.12, latitude: 13.13})})\")\n\n val res = df.select(\"location\").collectAsList().get(0).getAs[GenericRowWithSchema](0);\n\n assertEquals(\"point-2d\", res.get(0))\n assertEquals(4326, res.get(1))\n assertEquals(12.12, res.get(2))\n assertEquals(13.13, res.get(3))\n }\n\n @Test\n def testReadNodeWithPoint3D(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {location: point({x: 12.12, y: 13.13, z: 1})})\")\n\n val res = df.select(\"location\").collectAsList().get(0).getAs[GenericRowWithSchema](0)\n\n assertEquals(\"point-3d\", res.get(0))\n assertEquals(9157, res.get(1))\n assertEquals(12.12, res.get(2))\n assertEquals(13.13, res.get(3))\n assertEquals(1.0, res.get(4))\n }\n\n @Test\n def testReadNodeWithDate(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {born: date('2009-10-10')})\")\n\n val list = df.select(\"born\").collectAsList()\n val res = list.get(0).getDate(0)\n\n assertEquals(java.sql.Date.valueOf(\"2009-10-10\"), res)\n }\n\n @Test\n def testReadNodeWithDuration(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {range: duration({days: 14, hours:16, minutes: 12})})\")\n\n val list = df.select(\"range\").collectAsList()\n val res = list.get(0).getAs[GenericRowWithSchema](0)\n\n assertEquals(\"duration\", res(0))\n assertEquals(0L, res(1))\n assertEquals(14L, res(2))\n assertEquals(58320L, res(3))\n assertEquals(0, res(4))\n assertEquals(\"P0M14DT58320S\", res(5))\n }\n\n @Test\n def testReadNodeWithStringArray(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {names: ['John', 'Doe']})\")\n\n val res = df.select(\"names\").collectAsList().get(0).getAs[Seq[String]](0)\n\n assertEquals(\"John\", res.head)\n assertEquals(\"Doe\", res(1))\n }\n\n @Test\n def testReadNodeWithLongArray(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {ages: [22, 23]})\")\n\n val res = df.select(\"ages\").collectAsList().get(0).getAs[Seq[Long]](0)\n\n assertEquals(22, res.head)\n assertEquals(23, res(1))\n }\n\n @Test\n def testReadNodeWithDoubleArray(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {scores: [22.33, 44.55]})\")\n\n val res = df.select(\"scores\").collectAsList().get(0).getAs[Seq[Double]](0)\n\n assertEquals(22.33, res.head, 0)\n assertEquals(44.55, res(1), 0)\n }\n\n @Test\n def testReadNodeWithTimestampArray(): Unit = {\n val df: DataFrame =\n initTest(\n s\"CREATE (p:Person {someTimes: [datetime('2010-10-10T11:13:37+01:00'), datetime('2011-11-11T10:13:37Z')]})\"\n )\n\n val res = df.select(\"someTimes\").collectAsList().get(0).getAs[Seq[Timestamp]](0)\n assertEquals(\"2010-10-10T10:13:37Z\", res.head.toInstant.atZone(ZoneOffset.UTC).toString)\n assertEquals(\"2011-11-11T10:13:37Z\", res(1).toInstant.atZone(ZoneOffset.UTC).toString)\n }\n\n @Test\n def testReadNodeWithLocalTimeArray(): Unit = {\n val df: DataFrame =\n initTest(s\"CREATE (p:Person {someTimes: [localtime({hour:12}), localtime({hour:1, minute: 3})]})\")\n\n val res = df.select(\"someTimes\").collectAsList().get(0).getAs[Seq[GenericRowWithSchema]](0)\n assertEquals(\"local-time\", res.head.get(0))\n assertEquals(\"12:00:00\", res.head.get(1))\n assertEquals(\"local-time\", res(1).get(0))\n assertEquals(\"01:03:00\", res(1).get(1))\n }\n\n @Test\n def testReadNodeWithBooleanArray(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {bools: [true, false]})\")\n\n val res = df.select(\"bools\").collectAsList().get(0).getAs[Seq[Boolean]](0)\n\n assertEquals(true, res.head)\n assertEquals(false, res(1))\n }\n\n @Test\n def testReadNodeWithPointArray(): Unit = {\n val df: DataFrame =\n initTest(s\"CREATE (p:Person {locations: [point({x: 11, y: 33.111}), point({x: 22, y: 44.222})]})\")\n\n val res = df.select(\"locations\").collectAsList().get(0).getAs[Seq[GenericRowWithSchema]](0)\n\n assertEquals(\"point-2d\", res.head.get(0))\n assertEquals(7203, res.head.get(1))\n assertEquals(11.0, res.head.get(2))\n assertEquals(33.111, res.head.get(3))\n\n assertEquals(\"point-2d\", res(1).get(0))\n assertEquals(7203, res(1).get(1))\n assertEquals(22.0, res(1).get(2))\n assertEquals(44.222, res(1).get(3))\n }\n\n @Test\n def testReadNodeWithGeoPointArray(): Unit = {\n val df: DataFrame = initTest(\n s\"CREATE (p:Person {locations: [point({longitude: 11, latitude: 33.111}), point({longitude: 22, latitude: 44.222})]})\"\n )\n\n val res = df.select(\"locations\").collectAsList().get(0).getAs[Seq[GenericRowWithSchema]](0)\n\n assertEquals(\"point-2d\", res.head.get(0))\n assertEquals(4326, res.head.get(1))\n assertEquals(11.0, res.head.get(2))\n assertEquals(33.111, res.head.get(3))\n\n assertEquals(\"point-2d\", res(1).get(0))\n assertEquals(4326, res(1).get(1))\n assertEquals(22.0, res(1).get(2))\n assertEquals(44.222, res(1).get(3))\n }\n\n @Test\n def testReadNodeWithPoint3DArray(): Unit = {\n val df: DataFrame =\n initTest(s\"CREATE (p:Person {locations: [point({x: 11, y: 33.111, z: 12}), point({x: 22, y: 44.222, z: 99.1})]})\")\n\n val res = df.select(\"locations\").collectAsList().get(0).getAs[Seq[GenericRowWithSchema]](0)\n\n assertEquals(\"point-3d\", res.head.get(0))\n assertEquals(9157, res.head.get(1))\n assertEquals(11.0, res.head.get(2))\n assertEquals(33.111, res.head.get(3))\n assertEquals(12.0, res.head.get(4))\n\n assertEquals(\"point-3d\", res(1).get(0))\n assertEquals(9157, res(1).get(1))\n assertEquals(22.0, res(1).get(2))\n assertEquals(44.222, res(1).get(3))\n assertEquals(99.1, res(1).get(4))\n }\n\n @Test\n def testReadNodeWithArrayDate(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {dates: [date('2009-10-10'), date('2009-10-11')]})\")\n\n val res = df.select(\"dates\").collectAsList().get(0).getAs[Seq[java.sql.Date]](0)\n\n assertEquals(java.sql.Date.valueOf(\"2009-10-10\"), res.head)\n assertEquals(java.sql.Date.valueOf(\"2009-10-11\"), res(1))\n }\n\n @Test\n def testReadNodeWithArrayZonedDateTime(): Unit = {\n val datetime1 = \"2015-06-24T12:50:35.556+01:00\"\n val datetime2 = \"2015-06-23T12:50:35.556+01:00\"\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p:Person {aTime: [\n datetime('$datetime1'),\n datetime('$datetime2')\n ]})\n \"\"\"\n )\n\n val result = df.select(\"aTime\").collectAsList().get(0).getAs[Seq[Timestamp]](0)\n\n assertEquals(Timestamp.from(OffsetDateTime.parse(datetime1).toInstant), result.head)\n assertEquals(Timestamp.from(OffsetDateTime.parse(datetime2).toInstant), result(1))\n }\n\n @Test\n def testReadNodeWithArrayDurations(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {durations: [duration({months: 0.75}), duration({weeks: 2.5})]})\")\n\n val res = df.select(\"durations\").collectAsList().get(0).getAs[Seq[GenericRowWithSchema]](0)\n\n assertEquals(\"duration\", res.head.get(0))\n assertEquals(0L, res.head.get(1))\n assertEquals(22L, res.head.get(2))\n assertEquals(71509L, res.head.get(3))\n assertEquals(500000000, res.head.get(4))\n assertEquals(\"P0M22DT71509.500000000S\", res.head.get(5))\n\n assertEquals(\"duration\", res(1).get(0))\n assertEquals(0L, res(1).get(1))\n assertEquals(17L, res(1).get(2))\n assertEquals(43200L, res(1).get(3))\n assertEquals(0, res(1).get(4))\n assertEquals(\"P0M17DT43200S\", res(1).get(5))\n }\n\n @Test\n def testReadNodeWithBinary(): Unit = {\n val bytes = \"hello, world!\".map(_.toByte).toArray\n val parameters = new java.util.HashMap[String, Object]()\n parameters.put(\"bytes\", bytes)\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction((tx: Transaction) => tx.run(\"CREATE (h:Hello {b: $bytes})\", parameters).consume())\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Hello\")\n .load()\n\n val res = df.select(\"b\").collect()\n assertEquals(1, res.length)\n val gotBytes = res.head.getAs[Array[Byte]](0)\n\n for (i <- bytes.indices) {\n assertEquals(bytes(i), gotBytes(i))\n }\n }\n\n @Test\n def testReadNodeWithEqualToFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {name: 'John Doe'}),\n (p2:Person {name: 'Jane Doe'})\n \"\"\"\n )\n\n val result = df.select(\"name\").where(\"name = 'John Doe'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(\"John Doe\", result.get(0).getString(0))\n }\n\n @Test\n def testReadNodeWithEqualToDateFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {birth: date('1998-02-04')}),\n (p2:Person {birth: date('1988-01-05')})\n \"\"\"\n )\n\n val result = df.select(\"birth\").where(\"birth = '1988-01-05'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(java.sql.Date.valueOf(\"1988-01-05\"), result.get(0).getDate(0))\n }\n\n @Test\n def testReadNodeWithTimestampGteFilter(): Unit = {\n val localDateTime = \"2007-12-03T10:15:30\"\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {birth: localdatetime('$localDateTime')}),\n (p2:Person {birth: localdatetime('$localDateTime')})\n \"\"\"\n )\n\n df.printSchema()\n df.show()\n\n val result = df.select(\"birth\").where(s\"birth >= '$localDateTime'\").collectAsList()\n assertEquals(2, result.size())\n }\n\n @Test\n def testReadNodeWithNotEqualToFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {name: 'John Doe'}),\n (p2:Person {name: 'Jane Doe'})\n \"\"\"\n )\n\n val result = df.select(\"name\").where(\"NOT name = 'John Doe'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(\"Jane Doe\", result.get(0).getString(0))\n }\n\n @Test\n def testReadNodeWithNotEqualToDateFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {birth: date('1998-02-04')}),\n (p2:Person {birth: date('1988-01-05')})\n \"\"\"\n )\n\n val result = df.select(\"birth\").where(\"NOT birth = '1988-01-05'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(java.sql.Date.valueOf(\"1998-02-04\"), result.get(0).getDate(0))\n }\n\n @Test\n def testReadNodeWithDifferentOperatorFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {name: 'John Doe'}),\n (p2:Person {name: 'Jane Doe'})\n \"\"\"\n )\n\n val result = df.select(\"name\").where(\"name != 'John Doe'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(\"Jane Doe\", result.get(0).getString(0))\n }\n\n @Test\n def testReadNodeWithGtFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {age: 19}),\n (p2:Person {age: 20}),\n (p3:Person {age: 21})\n \"\"\"\n )\n\n val result = df.select(\"age\").where(\"age > 20\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(21, result.get(0).getLong(0))\n }\n\n @Test\n def testReadNodeWithGtDateFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {birth: date('1998-02-04')}),\n (p2:Person {birth: date('1988-01-05')}),\n (p3:Person {birth: date('1994-10-16')})\n \"\"\"\n )\n\n val result = df.select(\"birth\").orderBy(\"birth\").where(\"birth > '1990-01-01'\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(java.sql.Date.valueOf(\"1994-10-16\"), result.get(0).getDate(0))\n assertEquals(java.sql.Date.valueOf(\"1998-02-04\"), result.get(1).getDate(0))\n }\n\n @Test\n def testReadNodeWithGtSpatialFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p:Person {location: point({x: 12, y: 12})}),\n (p2:Person {location: point({x: -6, y: -6})})\n \"\"\"\n )\n\n val result = df.select(\"location\").where(\"location.x > 0\").collectAsList()\n val row = result.get(0).getAs[GenericRowWithSchema](0);\n\n assertEquals(1, result.size())\n\n assertEquals(\"point-2d\", row.get(0))\n assertEquals(7203, row.get(1))\n assertEquals(12.0, row.get(2))\n assertEquals(12.0, row.get(3))\n }\n\n @Test\n def testReadNodeWithGteFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {age: 19}),\n (p2:Person {age: 20}),\n (p3:Person {age: 21})\n \"\"\"\n )\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age >= 20\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(20, result.get(0).getLong(0))\n assertEquals(21, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithGteFilterWithProp(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {score: 19, limit: 20}),\n (p2:Person {score: 20, limit: 18}),\n (p3:Person {score: 21, limit: 12})\n \"\"\"\n )\n\n val result = df.select(\"score\").orderBy(\"score\").where(\"score >= limit\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(20, result.get(0).getLong(0))\n assertEquals(21, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithLtFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: 41}),\n (p3:Person {age: 43})\n \"\"\"\n )\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age < 40\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(39, result.get(0).getLong(0))\n }\n\n @Test\n def testReadNodeWithLteFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: 41}),\n (p3:Person {age: 43})\n \"\"\"\n )\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age <= 41\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(39, result.get(0).getLong(0))\n assertEquals(41, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithInFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: 41}),\n (p3:Person {age: 43})\n \"\"\"\n )\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age IN(41,43)\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(41, result.get(0).getLong(0))\n assertEquals(43, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithIsNullFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: null}),\n (p3:Person {age: 43})\n \"\"\"\n )\n\n val result = df.select(\"age\").where(\"age IS NULL\").collectAsList()\n\n assertEquals(1, result.size())\n assertNull(result.get(0).get(0))\n }\n\n @Test\n def testReadNodeWithIsNotNullFilter(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: null}),\n (p3:Person {age: 43})\n \"\"\"\n )\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age IS NOT NULL\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(39, result.get(0).getLong(0))\n assertEquals(43, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithOrCondition(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: null}),\n (p3:Person {age: 43})\n \"\"\"\n )\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age = 43 OR age = 39 OR age = 32\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(39, result.get(0).getLong(0))\n assertEquals(43, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithAndCondition(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: null}),\n (p3:Person {age: 43})\n \"\"\"\n )\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age >= 39 AND age <= 43\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(39, result.get(0).getLong(0))\n assertEquals(43, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithStartsWith(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {name: 'John Mayer'}),\n (p2:Person {name: 'John Scofield'}),\n (p3:Person {name: 'John Butler'})\n \"\"\"\n )\n\n val result = df.select(\"name\").orderBy(\"name\").where(\"name LIKE 'John%'\").collectAsList()\n\n assertEquals(3, result.size())\n assertEquals(\"John Butler\", result.get(0).getString(0))\n assertEquals(\"John Mayer\", result.get(1).getString(0))\n assertEquals(\"John Scofield\", result.get(2).getString(0))\n }\n\n @Test\n def testReadNodeWithEndsWith(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {name: 'John Mayer'}),\n (p2:Person {name: 'John Scofield'}),\n (p3:Person {name: 'John Butler'})\n \"\"\"\n )\n\n val result = df.select(\"name\").where(\"name LIKE '%Scofield'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(\"John Scofield\", result.get(0).getString(0))\n }\n\n @Test\n def testReadNodeWithContains(): Unit = {\n val df: DataFrame = initTest(\n s\"\"\"\n CREATE (p1:Person {name: 'John Mayer'}),\n (p2:Person {name: 'John Scofield'}),\n (p3:Person {name: 'John Butler'})\n \"\"\"\n )\n\n val result = df.select(\"name\").where(\"name LIKE '%ay%'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(\"John Mayer\", result.get(0).getString(0))\n }\n\n @Test\n def testRelFiltersWithMap(): Unit = {\n val fixtureQuery: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Person {id:id,ids:[id,id]}) WITH collect(p) as people\n |UNWIND people as p1\n |UNWIND range(1,10) as friend\n |WITH p1, people[(p1.id + friend) % size(people)] as p2\n |CREATE (p1)-[:KNOWS]->(p2)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship.nodes.map\", \"true\")\n .option(\"relationship\", \"KNOWS\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Person\")\n .load()\n\n assertEquals(1, df.filter(\"``.`id` = '14' AND ``.`id` = '16'\").count)\n }\n\n @Test\n def testRelFiltersWithoutMap(): Unit = {\n val fixtureQuery: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Person {id:id,ids:[id,id]}) WITH collect(p) as people\n |UNWIND people as p1\n |UNWIND range(1,10) as friend\n |WITH p1, people[(p1.id + friend) % size(people)] as p2\n |CREATE (p1)-[:KNOWS]->(p2)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"KNOWS\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Person\")\n .load()\n\n assertEquals(1, df.filter(\"`source.id` = 14 AND `target.id` = 16\").count)\n }\n\n @Test\n def testReadRelationshipFilters(): Unit = {\n val fixtureQuery: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Person {id:id,ids:[id,id]}) WITH collect(p) as people\n |UNWIND people as p1\n |UNWIND range(1,10) as friend\n |WITH p1, people[(p1.id + friend) % size(people)] as p2\n |CREATE (p1)-[:KNOWS]->(p2)\n |RETURN *\n \"\"\".stripMargin\n\n val df: DataFrame = initTest(fixtureQuery)\n val repartitionedDf = df.repartition(10)\n\n assertEquals(10, repartitionedDf.rdd.getNumPartitions)\n val numNode = repartitionedDf.collect().length\n assertEquals(100, numNode)\n }\n\n @Test\n def testRelationshipsDifferentFieldValues(): Unit = {\n val fixtureQuery: String =\n s\"\"\"CREATE (pr1:Product {id: '1'})\n |CREATE (pr2:Product {id: 2})\n |CREATE (pe1:Person {id: '3'})\n |CREATE (pe2:Person {id: 4})\n |CREATE (pe1)-[:BOUGHT]->(pr1)\n |CREATE (pe2)-[:BOUGHT]->(pr2)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n\n val res = df.sort(\"`source.id`\").collectAsList()\n\n assertEquals(\"3\", res.get(0).get(4))\n assertEquals(\"1\", res.get(0).get(7))\n assertEquals(\"4\", res.get(1).get(4))\n assertEquals(\"2\", res.get(1).get(7))\n }\n\n @Test\n def testReadNodesCustomPartitions(): Unit = {\n val fixtureQuery: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Person:Customer {id: id, name: 'Person ' + id})\n |RETURN *\n \"\"\".stripMargin\n val fixture2Query: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Employee:Customer {id: id, name: 'Person ' + id})\n |RETURN *\n \"\"\".stripMargin\n SparkConnectorScalaSuiteIT.driver.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n SparkConnectorScalaSuiteIT.driver.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixture2Query).consume()\n }\n )\n\n val partitionedDf = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":Person:Customer\")\n .option(\"partitions\", \"5\")\n .load()\n\n assertEquals(5, partitionedDf.rdd.getNumPartitions)\n assertEquals(100, partitionedDf.collect().map(_.getAs[Long](\"id\")).toSet.size)\n assertEquals(100, partitionedDf.collect().map(_.getAs[Long](\"id\")).size)\n }\n\n @Test\n def testReadRelsCustomPartitions(): Unit = {\n val fixtureQuery: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Person {id: id, name: 'Person ' + id})-[:BOUGHT{quantity: ceil(rand() * 100)}]->(:Product{id: id, name: 'Product ' + id})\n |RETURN *\n \"\"\".stripMargin\n SparkConnectorScalaSuiteIT.driver.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val partitionedDf = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship.nodes.map\", \"true\")\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .option(\"partitions\", \"5\")\n .load()\n\n assertEquals(5, partitionedDf.rdd.getNumPartitions)\n assertEquals(100, partitionedDf.collect().map(_.getAs[Long](\"\")).toSet.size)\n assertEquals(100, partitionedDf.collect().map(_.getAs[Long](\"\")).size)\n }\n\n @Test\n def testReadQueryCustomPartitions(): Unit = {\n val fixtureProduct1Query: String =\n \"\"\"CREATE (pr:Product{id: 1, name: 'Product 1'})\n |WITH pr\n |UNWIND range(1,100) as id\n |CREATE (p:Person {id: id, name: 'Person ' + id})-[:BOUGHT{quantity: ceil(rand() * 100)}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n SparkConnectorScalaSuiteIT.driver.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureProduct1Query).consume()\n }\n )\n val fixtureProduct2Query: String =\n \"\"\"CREATE (pr:Product{id: 2, name: 'Product 2'})\n |WITH pr\n |UNWIND range(1,50) as id\n |MATCH (p:Person {id: id})\n |CREATE (p)-[:BOUGHT{quantity: ceil(rand() * 100)}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n SparkConnectorScalaSuiteIT.driver.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureProduct2Query).consume()\n }\n )\n\n val partitionedQueryCountDf = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"\n |MATCH (p:Person)-[r:BOUGHT]->(pr:Product{name: 'Product 2'})\n |RETURN p.name AS person, pr.name AS product, r.quantity AS quantity\"\"\".stripMargin\n )\n .option(\"partitions\", \"5\")\n .option(\n \"query.count\",\n \"\"\"\n |MATCH (p:Person)-[r:BOUGHT]->(pr:Product{name: 'Product 2'})\n |RETURN count(p) AS count\"\"\".stripMargin\n )\n .load()\n\n assertEquals(5, partitionedQueryCountDf.rdd.getNumPartitions)\n assertEquals(50, partitionedQueryCountDf.collect().map(_.getAs[String](\"person\")).toSet.size)\n assertEquals(50, partitionedQueryCountDf.collect().map(_.getAs[String](\"person\")).length)\n\n val partitionedQueryCountLiteralDf = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"\n |MATCH (p:Person)-[r:BOUGHT]->(pr:Product{name: 'Product 2'})\n |RETURN p.name AS person, pr.name AS product, r.quantity AS quantity\"\"\".stripMargin\n )\n .option(\"partitions\", \"5\")\n .option(\"query.count\", \"50\")\n .load()\n\n assertEquals(5, partitionedQueryCountLiteralDf.rdd.getNumPartitions)\n assertEquals(50, partitionedQueryCountLiteralDf.collect().map(_.getAs[String](\"person\")).toSet.size)\n assertEquals(50, partitionedQueryCountLiteralDf.collect().map(_.getAs[String](\"person\")).length)\n }\n\n @Test\n def testRelationshipsFlatten(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n\n val count = df.collectAsList()\n .asScala\n .filter(row =>\n row.getAs[Long](\"\") >= 0\n && row.getAs[String](\"\") != null\n && row.getAs[Double](\"rel.when\") >= 0\n && row.getAs[Double](\"rel.quantity\") >= 0\n && row.getAs[Long](\"\") >= 0\n && row.getAs[Long](\"source.id\") >= 0\n && !row.getAs[Seq[String]](\"\").isEmpty\n && row.getAs[String](\"source.fullName\") != null\n && row.getAs[Long](\"\") >= 0\n && row.getAs[Double](\"target.id\") >= 0\n && !row.getAs[Seq[String]](\"\").isEmpty\n && row.getAs[String](\"target.name\") != null\n )\n .size\n assertEquals(total, count)\n }\n\n @Test\n def testRelationshipsMap(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship.nodes.map\", \"true\")\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n\n val rows = df.collectAsList().asScala\n val count = rows\n .filter(row =>\n row.getAs[Long](\"\") >= 0\n && row.getAs[String](\"\") != null\n && row.getAs[Double](\"rel.when\") >= 0\n && row.getAs[Double](\"rel.quantity\") >= 0\n && row.getAs[Map[String, String]](\"\") != null\n && row.getAs[Map[String, String]](\"\") != null\n )\n .size\n assertEquals(total, count)\n\n val countSourceMap = rows.map(row => row.getAs[Map[String, String]](\"\"))\n .filter(row => row.keys == Set(\"id\", \"fullName\", \"\", \"\"))\n .size\n assertEquals(total, countSourceMap)\n val countTargetMap = rows.map(row => row.getAs[Map[String, String]](\"\"))\n .filter(row => row.keys == Set(\"id\", \"name\", \"\", \"\"))\n .size\n assertEquals(total, countTargetMap)\n }\n\n @Test\n def testQueries(): Unit = {\n val dfMap = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"RETURN {a: 1, b: '3'} AS map\")\n .load()\n val map = dfMap.collect()(0).getAs[Map[String, String]](\"map\")\n val expectedMap = Map(\"a\" -> \"1\", \"b\" -> \"3\")\n assertEquals(expectedMap, map)\n\n val dfArrayMap = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"RETURN [{a: 1, b: '3'}, {a: 'foo'}] AS listMap\")\n .load()\n val listMap = dfArrayMap.collect()(0).getAs[Seq[_]](\"listMap\").toList\n val expectedListMap = Seq(Map(\"a\" -> \"1\", \"b\" -> \"3\"), Map(\"a\" -> \"foo\"))\n assertEquals(expectedListMap, listMap)\n\n val dfArray = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"RETURN [1, 'foo'] AS list\")\n .load()\n val list = dfArray.collect()(0).getAs[Seq[_]](\"list\")\n val expectedList = Seq(\"1\", \"foo\")\n assertEquals(expectedList, list)\n }\n\n @Test\n def testComplexQuery(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (n:Person) WITH n LIMIT 2 RETURN collect(n) AS nodes\")\n .load()\n\n val data = df.collect()\n val count = data.flatMap(row => row.getAs[Seq[Row]](\"nodes\"))\n .filter(row =>\n row.getAs[Long](\"\") >= 0\n && !row.getAs[Seq[String]](\"\").isEmpty\n && !row.getAs[String](\"fullName\").isEmpty\n && row.getAs[Long](\"id\") >= 0\n )\n .size\n assertEquals(2, count)\n\n val dfString: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"MATCH (p:Person)-[b:BOUGHT]->(pr:Product)\n |RETURN id(p) AS personId, id(pr) AS productId, {quantity: b.quantity, when: b.when} AS map\"\"\".stripMargin\n )\n .option(\"schema.strategy\", \"string\")\n .load()\n\n val dataString = dfString.collect()\n val countString = dataString\n .filter(row =>\n !row.getAs[String](\"personId\").isEmpty\n && !row.getAs[String](\"productId\").isEmpty\n && !row.getAs[String](\"map\").isEmpty\n )\n .size\n assertEquals(100, countString)\n\n val dfRel: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"MATCH (p:Person)-[b:BOUGHT]->(pr:Product)\n |RETURN b AS rel\"\"\".stripMargin\n )\n .load()\n val dataRel = dfRel.collect()\n val countRel = dataRel\n .map(_.getAs[Row](\"rel\"))\n .filter(row =>\n row.getAs[Long](\"\") >= 0\n && !row.getAs[String](\"\").isEmpty\n && row.getAs[Long](\"\") >= 0\n && row.getAs[Long](\"\") >= 0\n && row.getAs[Double](\"when\") != null\n && row.getAs[Double](\"quantity\") != null\n )\n .size\n assertEquals(100, countRel)\n }\n\n @Test\n def testShouldCreateTheCorrectDataframeWithTwoPartitions(): Unit = {\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary =\n tx.run(\"CREATE (i1:Instrument{name: 'Drums'}), (i2:Instrument{name: 'Guitar'})\").consume()\n }\n )\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Instrument\")\n .option(\"partitions\", \"2\")\n .load\n\n assertEquals(2, df.count())\n }\n\n @Test\n def testEmptyDataset(): Unit = {\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (e:ID_DO_NOT_EXIST) RETURN id(e) as f, 1 as g\")\n .load\n\n assertEquals(0, df.count())\n assertEquals(Set(\"f\", \"g\"), df.columns.toSet)\n }\n\n @Test\n def testColumnSorted(): Unit = {\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary =\n tx.run(\"CREATE (i1:Instrument{name: 'Drums', id: 1}), (i2:Instrument{name: 'Guitar', id: 2})\").consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (i:Instrument) RETURN id(i) as internal_id, i.id as id, i.name as name, i.name\")\n .load()\n .orderBy(\"id\")\n\n val row = df.collectAsList().get(0)\n assertEquals(1L, row.get(1))\n assertEquals(\"Drums\", row.get(2))\n assertEquals(Set(\"internal_id\", \"id\", \"name\", \"i.name\"), df.columns.toSet)\n }\n\n @Test\n def testComplexReturnStatement(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"MATCH (p:Person)-[b:BOUGHT]->(pr:Product)\n |RETURN id(p) AS personId, id(pr) AS productId, {quantity: b.quantity, when: b.when} AS map, \"some string\" as someString, {anotherField: \"201\"} as map2\"\"\".stripMargin\n )\n .option(\"schema.strategy\", \"string\")\n .load()\n\n assertEquals(Set(\"personId\", \"productId\", \"map\", \"someString\", \"map2\"), df.columns.toSet)\n assertEquals(100, df.count())\n }\n\n @Test\n def testComplexReturnStatementNoValues(): Unit = {\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"MATCH (p:Person)-[b:BOUGHT]->(pr:Product)\n |RETURN id(p) AS personId, id(pr) AS productId, {quantity: b.quantity, when: b.when} AS map, \"some string\" as someString, {anotherField: \"201\", and: 1} as map2\"\"\".stripMargin\n )\n .option(\"schema.strategy\", \"string\")\n .load()\n\n assertEquals(Set(\"personId\", \"productId\", \"map\", \"someString\", \"map2\"), df.columns.toSet)\n assertEquals(0, df.count())\n }\n\n @Test\n def testShouldPassTheScriptResult(): Unit = {\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"script\", \"RETURN 'foo' AS val\")\n .option(\"query\", \"UNWIND range(1,2) as id RETURN id AS val, scriptResult[0].val AS script\")\n .option(\"partitions\", 2)\n .option(\"query.count\", 2)\n .load\n .orderBy(\"val\")\n val data = df.collect()\n .map(row => (row.getAs[String](\"script\"), row.getAs[Long](\"val\")))\n .toSeq\n val expected = Seq((\"foo\", 1), (\"foo\", 2))\n assertEquals(expected, data)\n }\n\n @Test\n def testShouldFailWithExplicitErrorIfSkipLimitIsUsedAtTheEndOfTheQuery(): Unit = {\n try {\n ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (n:Label) RETURN id(n) as id LIMIT 100\")\n .option(\"partitions\", 2)\n .option(\"query.count\", 2)\n .load()\n .show() // we need the action to be able to trigger the exception because of the changes in Spark 3\n org.junit.Assert.fail(\"Expected to throw an exception\")\n } catch {\n case iae: IllegalArgumentException => {\n assertTrue(iae.getMessage.equals(\"SKIP/LIMIT are not allowed at the end of the query\"))\n }\n case _: Throwable => fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n\n @Test\n def testShouldFailWithExplicitErrorIfLowercaseSkipLimitIsUsedAtTheEndOfTheQuery(): Unit = {\n try {\n ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (n:Label) RETURN id(n) as id limit 100 skip 2\")\n .option(\"partitions\", 2)\n .option(\"query.count\", 2)\n .load()\n .show() // we need the action to be able to trigger the exception because of the changes in Spark 3\n org.junit.Assert.fail(\"Expected to throw an exception\")\n } catch {\n case iae: IllegalArgumentException => {\n assertTrue(iae.getMessage.equals(\"SKIP/LIMIT are not allowed at the end of the query\"))\n }\n case _: Throwable => fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n\n @Test\n def testShouldFailWithExplicitErrorIfRandomcaseSkipLimitIsUsedAtTheEndOfTheQuery(): Unit = {\n try {\n ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (n:Label) RETURN id(n) as id LiMIt 100 skIp 2\")\n .option(\"partitions\", 2)\n .option(\"query.count\", 2)\n .load()\n .show() // we need the action to be able to trigger the exception because of the changes in Spark 3\n org.junit.Assert.fail(\"Expected to throw an exception\")\n } catch {\n case iae: IllegalArgumentException => {\n assertTrue(iae.getMessage.equals(\"SKIP/LIMIT are not allowed at the end of the query\"))\n }\n case _: Throwable => fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n\n @Test\n def testShouldFailWithExplicitErrorIfSkipLimitIsUsedAtTheEndOfTheQueryWithMultilineQuery(): Unit = {\n try {\n ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"MATCH (n:Label)\\n\" +\n \"RETURN id(n) as id\\n\" +\n \"LIMIT 100\"\n )\n .option(\"partitions\", 2)\n .option(\"query.count\", 2)\n .load()\n .show() // we need the action to be able to trigger the exception because of the changes in Spark 3\n org.junit.Assert.fail(\"Expected to throw an exception\")\n } catch {\n case iae: IllegalArgumentException => {\n assertTrue(iae.getMessage.equals(\"SKIP/LIMIT are not allowed at the end of the query\"))\n }\n case t: Throwable => {\n t.printStackTrace()\n fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n }\n\n @Test\n def testShouldAllowSkipLimitInsideTheQuery(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id, name: 'Product ' + id})\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (p:Product) WITH p\\nLIMIT 10\\nRETURN p\")\n .option(\"partitions\", 2)\n .option(\"query.count\", 20)\n .load\n\n assertEquals(10, df.count())\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithNode(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id, name: 'Product ' + id})\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Product\")\n .load\n .select(\"name\")\n\n df.count()\n\n assertEquals(Set(\"name\"), df.columns.toSet)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithNodeAndWeirdColumnName(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id, `(╯°□°)╯︵ ┻━┻`: 'Product ' + id})\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Product\")\n .load\n .select(\"`(╯°□°)╯︵ ┻━┻`\")\n\n df.count()\n\n assertEquals(Set(\"(╯°□°)╯︵ ┻━┻\"), df.columns.toSet)\n }\n\n @Test\n def testShouldSelectTheSystemColumnsInRelationship(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Product\")\n .load\n .select(\"``\")\n\n df.collect()\n\n assertEquals(Set(\"\"), df.columns.toSet)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithRelationship(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Product\")\n .option(\"relationship.target.labels\", \"Person\")\n .load\n .select(\"`source.name`\", \"``\")\n\n df.count()\n\n assertEquals(Set(\"source.name\", \"\"), df.columns.toSet)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithRelationshipAndWeirdColumn(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), `(╯°□°)╯︵ ┻━┻`: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Product\")\n .load\n .select(\"`target.(╯°□°)╯︵ ┻━┻`\", \"``\")\n\n df.count()\n\n assertEquals(Set(\"target.(╯°□°)╯︵ ┻━┻\", \"\"), df.columns.toSet)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithQuery(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (p:Product) RETURN p.name as name\")\n .option(\"partitions\", 2)\n .option(\"query.count\", 20)\n .load\n .select(\"name\")\n\n df.count()\n\n assertEquals(Set(\"name\"), df.columns.toSet)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithFilter(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id, name: 'Product ' + id})\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Product\")\n .load\n .filter(\"name = 'Product 1'\")\n\n df.count()\n\n assertEquals(Set(\"\", \"\", \"name\", \"id\"), df.columns.toSet)\n }\n\n @Test\n def testShouldReturnJustTheSelectedFieldWithRelationshipWithFilter(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Product\")\n .load\n .filter(\"`target.name` = 'Product 1' AND `target.id` = '16'\")\n .select(\"`target.name`\", \"`target.id`\")\n\n df.count()\n\n assertEquals(Set(\"target.name\", \"target.id\"), df.columns.toSet)\n }\n\n @Test\n def testShouldUseTheUserSpecifiedSchema(): Unit = {\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary =\n tx.run(\"CREATE (p:Person {name: 'Foo Bar', age: 8})\").consume()\n }\n )\n\n val df = ss.read.format(classOf[DataSource].getName)\n .schema(StructType(Seq(StructField(\"age\", DataTypes.StringType)).toSeq))\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (n:Person) RETURN n.age AS age\")\n .load()\n\n assertEquals(\"8\", df.collect().head.get(0))\n }\n\n @Test\n def testQueryWithOrderByShouldBeAllowed(): Unit = {\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary =\n tx.run(\"CREATE (p:Person {name: 'Foo Bar', age: 8})\").consume()\n }\n )\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (n:Person) RETURN n.age AS age ORDER by age\")\n .load()\n\n assertEquals(8L, df.collect().head.get(0))\n }\n\n @Test\n def testShouldLimitTheNodeResults(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id, name: 'Product ' + id})\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction((tx: Transaction) => tx.run(fixtureQuery).consume())\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Product\")\n .load\n .limit(10)\n\n assertEquals(10, df.count())\n assertEquals(Set(\"\", \"\", \"name\", \"id\"), df.columns.toSet)\n }\n\n @Test\n def testShouldLimitTheRelationshipResults(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction((tx: Transaction) => tx.run(fixtureQuery).consume())\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Product\")\n .load\n .select(\"`target.name`\", \"`target.id`\")\n .limit(10)\n\n assertEquals(10, df.count())\n assertEquals(Set(\"target.name\", \"target.id\"), df.columns.toSet)\n }\n\n @Test\n def test531(): Unit = {\n val dataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"\n |UNWIND [\n | {first: '2022-06-14T10:02:28.192Z', second: null},\n | {first: '2022-06-15T10:02:28.192Z', second: '2022-06-16T10:02:28.192Z'}\n |]AS event\n |RETURN datetime(event.first) AS first, datetime(event.second) AS second\n |\"\"\".stripMargin\n )\n .load()\n\n assertEquals(\n StructType(Array(\n StructField(\"first\", DataTypes.TimestampType),\n StructField(\"second\", DataTypes.StringType)\n )),\n dataFrame.schema\n )\n\n assertEquals(\n List(\n (Timestamp.from(OffsetDateTime.parse(\"2022-06-14T10:02:28.192Z\").toInstant), null),\n (Timestamp.from(OffsetDateTime.parse(\"2022-06-15T10:02:28.192Z\").toInstant), \"2022-06-16T10:02:28.192Z\")\n ),\n dataFrame.collect()\n .map(r => (r.getTimestamp(0), r.getString(1)))\n .toList\n )\n }\n\n @Test\n def test531WithSchema(): Unit = {\n val schema = StructType(Array(\n StructField(\"first\", DataTypes.TimestampType),\n StructField(\"second\", DataTypes.TimestampType)\n ))\n // dataFrameWithSchema must with the proper data type\n val dataFrameWithSchema = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"\n |UNWIND [\n | {first: '2022-06-14T10:02:28.192Z', second: null},\n | {first: '2022-06-15T10:02:28.192Z', second: '2022-06-16T10:02:28.192Z'}\n |] AS event\n |RETURN datetime(event.first) AS first, datetime(event.second) AS second\n |\"\"\".stripMargin\n )\n .schema(schema)\n .load()\n assertEquals(\n schema,\n dataFrameWithSchema.schema\n )\n assertEquals(\n List(\n (Timestamp.from(OffsetDateTime.parse(\"2022-06-14T10:02:28.192Z\").toInstant), null),\n (\n Timestamp.from(OffsetDateTime.parse(\"2022-06-15T10:02:28.192Z\").toInstant),\n Timestamp.from(OffsetDateTime.parse(\"2022-06-16T10:02:28.192Z\").toInstant)\n )\n ),\n dataFrameWithSchema.collect()\n .map(r => (r.getTimestamp(0), r.getTimestamp(1)))\n .toList\n )\n }\n\n @Test\n def testShouldAggregateAndLimitTheRelationshipResults(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction((tx: Transaction) => tx.run(fixtureQuery).consume())\n\n val df = ss.read\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Product\")\n .load\n .select(\"`target.name`\", \"`target.id`\")\n .orderBy(col(\"`target.name`\").desc)\n .limit(10)\n\n df.show()\n assertEquals(10, df.count())\n assertEquals(Set(\"target.name\", \"target.id\"), df.columns.toSet)\n }\n\n private def initTest(query: String): DataFrame = {\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(query).consume()\n }\n )\n\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .load()\n }\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DataSourceReaderWithApocTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.sql.DataFrame\nimport org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema\nimport org.junit.Assert._\nimport org.junit.Test\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.summary.ResultSummary\n\nimport java.sql.Timestamp\nimport java.time.Instant\nimport java.time.LocalDateTime\nimport java.time.LocalTime\nimport java.time.OffsetDateTime\nimport java.time.OffsetTime\nimport java.time.ZoneOffset\nimport java.util.TimeZone\n\nimport scala.collection.JavaConverters._\nimport scala.collection.mutable.ArraySeq\nimport scala.collection.mutable.Seq\n\nclass DataSourceReaderWithApocTSE extends SparkConnectorScalaBaseWithApocTSE {\n\n @Test\n def testReadNodeHasIdField(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {name: 'John'})\")\n\n /**\n * utnaf: Since we can't be sure we are in total isolation, and the id is generated\n * internally by org.neo4j.neo4j, we just check that the field is an integer and is greater\n * than -1\n */\n assertTrue(df.select(\"\").collectAsList().get(0).getLong(0) > -1)\n }\n\n @Test\n def testReadNodeHasLabelsField(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person:Customer {name: 'John'})\")\n\n val result = df.select(\"\").collectAsList().get(0).getAs[Seq[String]](0)\n\n assertEquals(\"Person\", result.head)\n assertEquals(\"Customer\", result(1))\n }\n\n @Test\n def testReadNodeHasUnusualLabelsField(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:`Foo Bar`:Person {name: 'John'})\")\n\n val result = df.select(\"\").collectAsList().get(0).getAs[Seq[String]](0)\n\n assertEquals(Set(\"Person\", \"Foo Bar\"), result.toSet[String])\n }\n\n @Test\n def testReadNodeWithFieldWithDifferentTypes(): Unit = {\n val df: DataFrame = initTest(\"CREATE (p1:Person {id: 1, field: [12,34]}), (p2:Person {id: 2, field: 123})\")\n\n val res = df.orderBy(\"id\").collectAsList()\n\n assertEquals(\"[12,34]\", res.get(0).get(3))\n assertEquals(\"123\", res.get(1).get(3))\n }\n\n @Test\n def testReadNodeWithString(): Unit = {\n val name: String = \"John\"\n val df: DataFrame = initTest(s\"CREATE (p:Person {name: '$name'})\")\n\n assertEquals(name, df.select(\"name\").collectAsList().get(0).getString(0))\n }\n\n @Test\n def testReadNodeWithLong(): Unit = {\n val age: Long = 42\n val df: DataFrame = initTest(s\"CREATE (p:Person {age: $age})\")\n\n assertEquals(age, df.select(\"age\").collectAsList().get(0).getLong(0))\n }\n\n @Test\n def testReadNodeWithDouble(): Unit = {\n val score: Double = 3.14\n val df: DataFrame = initTest(s\"CREATE (p:Person {score: $score})\")\n\n assertEquals(score, df.select(\"score\").collectAsList().get(0).getDouble(0), 0)\n }\n\n @Test\n def testReadNodeWithLocalTime(): Unit = {\n val df: DataFrame =\n initTest(s\"CREATE (p:Person {aTime: localtime({hour:12, minute: 23, second: 0, millisecond: 294})})\")\n\n val result = df.select(\"aTime\").collectAsList().get(0).getAs[GenericRowWithSchema](0)\n\n assertEquals(\"local-time\", result.get(0))\n assertEquals(\"12:23:00.294\", result.get(1))\n }\n\n @Test\n def testReadNodeWithTime(): Unit = {\n val timezone = TimeZone.getDefault\n val df: DataFrame = initTest(s\"CREATE (p:Person {aTime: time({hour:12, minute: 23, second: 0, millisecond: 294})})\")\n\n val result = df.select(\"aTime\").collectAsList().get(0).getAs[GenericRowWithSchema](0)\n\n val localTime = LocalTime.of(12, 23, 0, 294000000)\n val expectedTime = OffsetTime.of(localTime, timezone.toZoneId.getRules.getOffset(Instant.now()))\n\n assertEquals(\"offset-time\", result.get(0))\n assertEquals(expectedTime.toString, result.get(1))\n }\n\n @Test\n def testReadNodeWithLocalDateTime(): Unit = {\n val localDateTime = \"2007-12-03T10:15:30\"\n val df: DataFrame = initTest(s\"CREATE (p:Person {aTime: localdatetime('$localDateTime')})\")\n\n val result = df.select(\"aTime\").collectAsList().get(0).getAs[LocalDateTime](0)\n assertEquals(LocalDateTime.parse(localDateTime), result)\n }\n\n @Test\n def testReadNodeWithZonedDateTime(): Unit = {\n val datetime = \"2015-06-24T12:50:35.556+01:00\"\n val df: DataFrame = initTest(s\"CREATE (p:Person {aTime: datetime('$datetime')})\")\n\n val result = df.select(\"aTime\").collectAsList().get(0).getTimestamp(0)\n\n assertEquals(Timestamp.from(OffsetDateTime.parse(datetime).toInstant), result)\n }\n\n @Test\n def testReadNodeWithPoint(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {location: point({x: 12.12, y: 13.13})})\")\n\n val res = df.select(\"location\").collectAsList().get(0).getAs[GenericRowWithSchema](0);\n\n assertEquals(\"point-2d\", res.get(0))\n assertEquals(7203, res.get(1))\n assertEquals(12.12, res.get(2))\n assertEquals(13.13, res.get(3))\n }\n\n @Test\n def testReadNodeWithGeoPoint(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {location: point({longitude: 12.12, latitude: 13.13})})\")\n\n val res = df.select(\"location\").collectAsList().get(0).getAs[GenericRowWithSchema](0);\n\n assertEquals(\"point-2d\", res.get(0))\n assertEquals(4326, res.get(1))\n assertEquals(12.12, res.get(2))\n assertEquals(13.13, res.get(3))\n }\n\n @Test\n def testReadNodeWithPoint3D(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {location: point({x: 12.12, y: 13.13, z: 1})})\")\n\n val res = df.select(\"location\").collectAsList().get(0).getAs[GenericRowWithSchema](0)\n\n assertEquals(\"point-3d\", res.get(0))\n assertEquals(9157, res.get(1))\n assertEquals(12.12, res.get(2))\n assertEquals(13.13, res.get(3))\n assertEquals(1.0, res.get(4))\n }\n\n @Test\n def testReadNodeWithDate(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {born: date('2009-10-10')})\")\n\n val list = df.select(\"born\").collectAsList()\n val res = list.get(0).getDate(0)\n\n assertEquals(java.sql.Date.valueOf(\"2009-10-10\"), res)\n }\n\n @Test\n def testReadNodeWithDuration(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {range: duration({days: 14, hours:16, minutes: 12})})\")\n\n val list = df.select(\"range\").collectAsList()\n val res = list.get(0).getAs[GenericRowWithSchema](0)\n\n assertEquals(\"duration\", res(0))\n assertEquals(0L, res(1))\n assertEquals(14L, res(2))\n assertEquals(58320L, res(3))\n assertEquals(0, res(4))\n assertEquals(\"P0M14DT58320S\", res(5))\n }\n\n @Test\n def testReadNodeWithStringArray(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {names: ['John', 'Doe']})\")\n\n val res = df.select(\"names\").collectAsList().get(0).getAs[Seq[String]](0)\n\n assertEquals(\"John\", res.head)\n assertEquals(\"Doe\", res(1))\n }\n\n @Test\n def testReadNodeWithLongArray(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {ages: [22, 23]})\")\n\n val res = df.select(\"ages\").collectAsList().get(0).getAs[Seq[Long]](0)\n\n assertEquals(22, res.head)\n assertEquals(23, res(1))\n }\n\n @Test\n def testReadNodeWithDoubleArray(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {scores: [22.33, 44.55]})\")\n\n val res = df.select(\"scores\").collectAsList().get(0).getAs[Seq[Double]](0)\n\n assertEquals(22.33, res.head, 0)\n assertEquals(44.55, res(1), 0)\n }\n\n @Test\n def testReadNodeWithLocalTimeArray(): Unit = {\n val df: DataFrame =\n initTest(s\"CREATE (p:Person {someTimes: [localtime({hour:12}), localtime({hour:1, minute: 3})]})\")\n\n val res = df.select(\"someTimes\").collectAsList().get(0).getAs[Seq[GenericRowWithSchema]](0)\n\n assertEquals(\"local-time\", res.head.get(0))\n assertEquals(\"12:00:00\", res.head.get(1))\n assertEquals(\"local-time\", res(1).get(0))\n assertEquals(\"01:03:00\", res(1).get(1))\n }\n\n @Test\n def testReadNodeWithBooleanArray(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {bools: [true, false]})\")\n\n val res = df.select(\"bools\").collectAsList().get(0).getAs[Seq[Boolean]](0)\n\n assertEquals(true, res.head)\n assertEquals(false, res(1))\n }\n\n @Test\n def testReadNodeWithPointArray(): Unit = {\n val df: DataFrame =\n initTest(s\"CREATE (p:Person {locations: [point({x: 11, y: 33.111}), point({x: 22, y: 44.222})]})\")\n\n val res = df.select(\"locations\").collectAsList().get(0).getAs[Seq[GenericRowWithSchema]](0)\n\n assertEquals(\"point-2d\", res.head.get(0))\n assertEquals(7203, res.head.get(1))\n assertEquals(11.0, res.head.get(2))\n assertEquals(33.111, res.head.get(3))\n\n assertEquals(\"point-2d\", res(1).get(0))\n assertEquals(7203, res(1).get(1))\n assertEquals(22.0, res(1).get(2))\n assertEquals(44.222, res(1).get(3))\n }\n\n @Test\n def testReadNodeWithGeoPointArray(): Unit = {\n val df: DataFrame = initTest(\n s\"CREATE (p:Person {locations: [point({longitude: 11, latitude: 33.111}), point({longitude: 22, latitude: 44.222})]})\"\n )\n\n val res = df.select(\"locations\").collectAsList().get(0).getAs[Seq[GenericRowWithSchema]](0)\n\n assertEquals(\"point-2d\", res.head.get(0))\n assertEquals(4326, res.head.get(1))\n assertEquals(11.0, res.head.get(2))\n assertEquals(33.111, res.head.get(3))\n\n assertEquals(\"point-2d\", res(1).get(0))\n assertEquals(4326, res(1).get(1))\n assertEquals(22.0, res(1).get(2))\n assertEquals(44.222, res(1).get(3))\n }\n\n @Test\n def testReadNodeWithPoint3DArray(): Unit = {\n val df: DataFrame =\n initTest(s\"CREATE (p:Person {locations: [point({x: 11, y: 33.111, z: 12}), point({x: 22, y: 44.222, z: 99.1})]})\")\n\n val res = df.select(\"locations\").collectAsList().get(0).getAs[Seq[GenericRowWithSchema]](0)\n\n assertEquals(\"point-3d\", res.head.get(0))\n assertEquals(9157, res.head.get(1))\n assertEquals(11.0, res.head.get(2))\n assertEquals(33.111, res.head.get(3))\n assertEquals(12.0, res.head.get(4))\n\n assertEquals(\"point-3d\", res(1).get(0))\n assertEquals(9157, res(1).get(1))\n assertEquals(22.0, res(1).get(2))\n assertEquals(44.222, res(1).get(3))\n assertEquals(99.1, res(1).get(4))\n }\n\n @Test\n def testReadNodeWithArrayDate(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {dates: [date('2009-10-10'), date('2009-10-11')]})\")\n\n val res = df.select(\"dates\").collectAsList().get(0).getAs[Seq[java.sql.Date]](0)\n\n assertEquals(java.sql.Date.valueOf(\"2009-10-10\"), res.head)\n assertEquals(java.sql.Date.valueOf(\"2009-10-11\"), res(1))\n }\n\n @Test\n def testReadNodeWithArrayZonedDateTime(): Unit = {\n val datetime1 = \"2015-06-24T12:50:35.556+01:00\"\n val datetime2 = \"2015-06-23T12:50:35.556+01:00\"\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p:Person {aTime: [\n datetime('$datetime1'),\n datetime('$datetime2')\n ]})\n \"\"\")\n\n val result = df.select(\"aTime\").collectAsList().get(0).getAs[Seq[Timestamp]](0)\n\n assertEquals(Timestamp.from(OffsetDateTime.parse(datetime1).toInstant), result.head)\n assertEquals(Timestamp.from(OffsetDateTime.parse(datetime2).toInstant), result(1))\n }\n\n @Test\n def testReadNodeWithArrayDurations(): Unit = {\n val df: DataFrame = initTest(s\"CREATE (p:Person {durations: [duration({months: 0.75}), duration({weeks: 2.5})]})\")\n\n val res = df.select(\"durations\").collectAsList().get(0).getAs[Seq[GenericRowWithSchema]](0)\n\n assertEquals(\"duration\", res.head.get(0))\n assertEquals(0L, res.head.get(1))\n assertEquals(22L, res.head.get(2))\n assertEquals(71509L, res.head.get(3))\n assertEquals(500000000, res.head.get(4))\n assertEquals(\"P0M22DT71509.500000000S\", res.head.get(5))\n\n assertEquals(\"duration\", res(1).get(0))\n assertEquals(0L, res(1).get(1))\n assertEquals(17L, res(1).get(2))\n assertEquals(43200L, res(1).get(3))\n assertEquals(0, res(1).get(4))\n assertEquals(\"P0M17DT43200S\", res(1).get(5))\n }\n\n @Test\n def testReadNodeWithEqualToFilter(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {name: 'John Doe'}),\n (p2:Person {name: 'Jane Doe'})\n \"\"\")\n\n val result = df.select(\"name\").where(\"name = 'John Doe'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(\"John Doe\", result.get(0).getString(0))\n }\n\n @Test\n def testReadNodeWithNotEqualToFilter(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {name: 'John Doe'}),\n (p2:Person {name: 'Jane Doe'})\n \"\"\")\n\n val result = df.select(\"name\").where(\"NOT name = 'John Doe'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(\"Jane Doe\", result.get(0).getString(0))\n }\n\n @Test\n def testReadNodeWithDifferentOperatorFilter(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {name: 'John Doe'}),\n (p2:Person {name: 'Jane Doe'})\n \"\"\")\n\n val result = df.select(\"name\").where(\"name != 'John Doe'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(\"Jane Doe\", result.get(0).getString(0))\n }\n\n @Test\n def testReadNodeWithGtFilter(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {age: 19}),\n (p2:Person {age: 20}),\n (p3:Person {age: 21})\n \"\"\")\n\n val result = df.select(\"age\").where(\"age > 20\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(21, result.get(0).getLong(0))\n }\n\n @Test\n def testReadNodeWithGteFilter(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {age: 19}),\n (p2:Person {age: 20}),\n (p3:Person {age: 21})\n \"\"\")\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age >= 20\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(20, result.get(0).getLong(0))\n assertEquals(21, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithGteFilterWithProp(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {score: 19, limit: 20}),\n (p2:Person {score: 20, limit: 18}),\n (p3:Person {score: 21, limit: 12})\n \"\"\")\n\n val result = df.select(\"score\").orderBy(\"score\").where(\"score >= limit\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(20, result.get(0).getLong(0))\n assertEquals(21, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithLtFilter(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: 41}),\n (p3:Person {age: 43})\n \"\"\")\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age < 40\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(39, result.get(0).getLong(0))\n }\n\n @Test\n def testReadNodeWithLteFilter(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: 41}),\n (p3:Person {age: 43})\n \"\"\")\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age <= 41\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(39, result.get(0).getLong(0))\n assertEquals(41, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithInFilter(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: 41}),\n (p3:Person {age: 43})\n \"\"\")\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age IN(41,43)\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(41, result.get(0).getLong(0))\n assertEquals(43, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithIsNullFilter(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: null}),\n (p3:Person {age: 43})\n \"\"\")\n\n val result = df.select(\"age\").where(\"age IS NULL\").collectAsList()\n\n assertEquals(1, result.size())\n assertNull(result.get(0).get(0))\n }\n\n @Test\n def testReadNodeWithIsNotNullFilter(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: null}),\n (p3:Person {age: 43})\n \"\"\")\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age IS NOT NULL\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(39, result.get(0).getLong(0))\n assertEquals(43, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithOrCondition(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: null}),\n (p3:Person {age: 43})\n \"\"\")\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age = 43 OR age = 39 OR age = 32\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(39, result.get(0).getLong(0))\n assertEquals(43, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithAndCondition(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {age: 39}),\n (p2:Person {age: null}),\n (p3:Person {age: 43})\n \"\"\")\n\n val result = df.select(\"age\").orderBy(\"age\").where(\"age >= 39 AND age <= 43\").collectAsList()\n\n assertEquals(2, result.size())\n assertEquals(39, result.get(0).getLong(0))\n assertEquals(43, result.get(1).getLong(0))\n }\n\n @Test\n def testReadNodeWithStartsWith(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {name: 'John Mayer'}),\n (p2:Person {name: 'John Scofield'}),\n (p3:Person {name: 'John Butler'})\n \"\"\")\n\n val result = df.select(\"name\").orderBy(\"name\").where(\"name LIKE 'John%'\").collectAsList()\n\n assertEquals(3, result.size())\n assertEquals(\"John Butler\", result.get(0).getString(0))\n assertEquals(\"John Mayer\", result.get(1).getString(0))\n assertEquals(\"John Scofield\", result.get(2).getString(0))\n }\n\n @Test\n def testReadNodeWithEndsWith(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {name: 'John Mayer'}),\n (p2:Person {name: 'John Scofield'}),\n (p3:Person {name: 'John Butler'})\n \"\"\")\n\n val result = df.select(\"name\").where(\"name LIKE '%Scofield'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(\"John Scofield\", result.get(0).getString(0))\n }\n\n @Test\n def testReadNodeWithContains(): Unit = {\n val df: DataFrame = initTest(s\"\"\"\n CREATE (p1:Person {name: 'John Mayer'}),\n (p2:Person {name: 'John Scofield'}),\n (p3:Person {name: 'John Butler'})\n \"\"\")\n\n val result = df.select(\"name\").where(\"name LIKE '%ay%'\").collectAsList()\n\n assertEquals(1, result.size())\n assertEquals(\"John Mayer\", result.get(0).getString(0))\n }\n\n @Test\n def testRelFiltersWithMap(): Unit = {\n val fixtureQuery: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Person {id:id,ids:[id,id]}) WITH collect(p) as people\n |UNWIND people as p1\n |UNWIND range(1,10) as friend\n |WITH p1, people[(p1.id + friend) % size(people)] as p2\n |CREATE (p1)-[:KNOWS]->(p2)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteWithApocIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"relationship.nodes.map\", \"true\")\n .option(\"relationship\", \"KNOWS\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Person\")\n .load()\n\n assertEquals(1, df.filter(\"``.`id` = '14' AND ``.`id` = '16'\").collectAsList().size())\n }\n\n @Test\n def testRelFiltersWithoutMap(): Unit = {\n val fixtureQuery: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Person {id:id,ids:[id,id]}) WITH collect(p) as people\n |UNWIND people as p1\n |UNWIND range(1,10) as friend\n |WITH p1, people[(p1.id + friend) % size(people)] as p2\n |CREATE (p1)-[:KNOWS]->(p2)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteWithApocIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"relationship\", \"KNOWS\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Person\")\n .load()\n\n assertEquals(1, df.filter(\"`source.id` = 14 AND `target.id` = 16\").collectAsList().size())\n }\n\n @Test\n def testReadNodeRepartition(): Unit = {\n val fixtureQuery: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Person {id:id,ids:[id,id]}) WITH collect(p) as people\n |UNWIND people as p1\n |UNWIND range(1,10) as friend\n |WITH p1, people[(p1.id + friend) % size(people)] as p2\n |CREATE (p1)-[:KNOWS]->(p2)\n |RETURN *\n \"\"\".stripMargin\n\n val df: DataFrame = initTest(fixtureQuery)\n val repartitionedDf = df.repartition(10)\n\n assertEquals(10, repartitionedDf.rdd.getNumPartitions)\n val numNode = repartitionedDf.collect().length\n assertEquals(100, numNode)\n }\n\n @Test\n def testRelationshipsFlatten(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteWithApocIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n\n val count = df.collectAsList()\n .asScala\n .filter(row =>\n row.getAs[Long](\"\") >= 0\n && row.getAs[String](\"\") != null\n && row.getAs[Double](\"rel.when\") >= 0\n && row.getAs[Double](\"rel.quantity\") >= 0\n && row.getAs[Long](\"\") >= 0\n && row.getAs[Long](\"source.id\") >= 0\n && !row.getAs[Seq[String]](\"\").isEmpty\n && row.getAs[String](\"source.fullName\") != null\n && row.getAs[Long](\"\") >= 0\n && row.getAs[Double](\"target.id\") >= 0\n && !row.getAs[Seq[String]](\"\").isEmpty\n && row.getAs[String](\"target.name\") != null\n )\n .size\n assertEquals(total, count)\n }\n\n @Test\n def testRelationshipsMap(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * rand(), name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteWithApocIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.nodes.map\", \"true\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n\n val rows = df.collectAsList().asScala\n val count = rows\n .filter(row =>\n row.getAs[Long](\"\") >= 0\n && row.getAs[String](\"\") != null\n && row.getAs[Double](\"rel.when\") >= 0\n && row.getAs[Double](\"rel.quantity\") >= 0\n && row.getAs[Map[String, String]](\"\") != null\n && row.getAs[Map[String, String]](\"\") != null\n )\n .size\n assertEquals(total, count)\n\n val countSourceMap = rows.map(row => row.getAs[Map[String, String]](\"\"))\n .filter(row => row.keys == Set(\"id\", \"fullName\", \"\", \"\"))\n .size\n assertEquals(total, countSourceMap)\n val countTargetMap = rows.map(row => row.getAs[Map[String, String]](\"\"))\n .filter(row => row.keys == Set(\"id\", \"name\", \"\", \"\"))\n .size\n assertEquals(total, countTargetMap)\n }\n\n @Test\n def testRelationshipsDifferentFieldValues(): Unit = {\n val fixtureQuery: String =\n s\"\"\"CREATE (pr1:Product {id: '1'})\n |CREATE (pr2:Product {id: 2})\n |CREATE (pe1:Person {id: '3'})\n |CREATE (pe2:Person {id: 4})\n |CREATE (pe1)-[:BOUGHT]->(pr1)\n |CREATE (pe2)-[:BOUGHT]->(pr2)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteWithApocIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n\n val res = df.sort(\"`source.id`\").collectAsList()\n\n assertEquals(\"3\", res.get(0).get(4))\n assertEquals(\"1\", res.get(0).get(7))\n assertEquals(\"4\", res.get(1).get(4))\n assertEquals(\"2\", res.get(1).get(7))\n }\n\n @Test\n def testShouldReturnSamePropertiesForNodesWithMultipleLabels(): Unit = {\n val fixtureQuery: String =\n s\"\"\"CREATE (actor:Person:Actor {name: 'Keanu Reeves', born: 1964, actor: true})\n |CREATE (soccerPlayer:Person:SoccerPlayer {name: 'Zlatan Ibrahimović', born: 1981, soccerPlayer: true})\n |CREATE (writer:Person:Writer {name: 'Philip K. Dick', born: 1928, writer: true})\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteWithApocIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .load()\n .sort(\"name\")\n\n val cols = df.columns.toSeq.sorted\n val expectedCols = Seq(\"name\", \"born\", \"actor\", \"soccerPlayer\", \"writer\", \"\", \"\")\n .sorted\n assertEquals(expectedCols, cols)\n\n val data = df.collect().toSeq\n .map(row =>\n expectedCols.filterNot(_ == \"\").map(col => {\n row.getAs[Any](col) match {\n case array: Array[String] => array.toList\n case null => null\n case other: Any => other\n }\n })\n )\n val expectedData = Seq(\n Seq(ArraySeq(\"Person\", \"Actor\"), true, 1964, \"Keanu Reeves\", null, null),\n Seq(ArraySeq(\"Person\", \"Writer\"), null, 1928, \"Philip K. Dick\", null, true),\n Seq(ArraySeq(\"Person\", \"SoccerPlayer\"), null, 1981, \"Zlatan Ibrahimović\", true, null)\n ).toBuffer\n assertEquals(expectedData, data)\n }\n\n @Test\n def testShouldReturnSamePropertiesForNodesWithMultipleLabelsAndDifferentValues(): Unit = {\n val fixtureQuery: String =\n s\"\"\"CREATE (:Person { prop: 25 }),\n |(:Person:Player { prop: \"hello\" }),\n |(:Person:Player:Weirdo { prop: true })\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteWithApocIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val df: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .load()\n .sort(\"prop\")\n\n val cols = df.columns.toSeq.sorted\n val expectedCols = Seq(\"prop\", \"\", \"\")\n .sorted\n assertEquals(expectedCols, cols)\n\n val data = df.collect().toSeq\n .map(row =>\n expectedCols.filterNot(_ == \"\").map(col => {\n row.getAs[Any](col) match {\n case array: Array[String] => array.toList\n case null => null\n case other: Any => other\n }\n })\n )\n val expectedData = Seq(\n Seq(ArraySeq(\"Person\"), \"25\"),\n Seq(ArraySeq(\"Person\", \"Player\"), \"hello\"),\n Seq(ArraySeq(\"Person\", \"Player\", \"Weirdo\"), \"true\")\n )\n assertEquals(expectedData, data)\n }\n\n @Test\n def testReadNodesCustomPartitions(): Unit = {\n val fixtureQuery: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Person:Customer {id: id, name: 'Person ' + id})\n |RETURN *\n \"\"\".stripMargin\n val fixture2Query: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Employee:Customer {id: id, name: 'Person ' + id})\n |RETURN *\n \"\"\".stripMargin\n SparkConnectorScalaSuiteWithApocIT.driver.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n SparkConnectorScalaSuiteWithApocIT.driver.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixture2Query).consume()\n }\n )\n\n val partitionedDf = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"labels\", \":Person:Customer\")\n .option(\"partitions\", \"5\")\n .load()\n\n assertEquals(5, partitionedDf.rdd.getNumPartitions)\n assertEquals(100, partitionedDf.collect().map(_.getAs[Long](\"id\")).toSet.size)\n }\n\n @Test\n def testReadRelsCustomPartitions(): Unit = {\n val fixtureQuery: String =\n \"\"\"UNWIND range(1,100) as id\n |CREATE (p:Person {id: id, name: 'Person ' + id})-[:BOUGHT{quantity: ceil(rand() * 100)}]->(:Product{id: id, name: 'Product ' + id})\n |RETURN *\n \"\"\".stripMargin\n SparkConnectorScalaSuiteWithApocIT.driver.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val partitionedDf = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"relationship.nodes.map\", \"true\")\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .option(\"partitions\", \"5\")\n .load()\n\n assertEquals(5, partitionedDf.rdd.getNumPartitions)\n assertEquals(100, partitionedDf.collect().map(_.getAs[Long](\"\")).toSet.size)\n }\n\n @Test\n def testReturnProcedure(): Unit = {\n val query =\n \"\"\"RETURN apoc.convert.toSet([1,1,3]) AS foo, 'bar' AS bar\n |\"\"\".stripMargin\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"partitions\", 1)\n .option(\"query\", query)\n .load\n\n assertEquals(Set(\"foo\", \"bar\"), df.columns.toSet)\n assertEquals(1, df.count())\n }\n\n private def initTest(query: String): DataFrame = {\n SparkConnectorScalaSuiteWithApocIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(query).consume()\n }\n )\n\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithApocIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .load()\n }\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DataSourceSchemaWriterTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.sql.DataFrame\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.SparkSession\nimport org.apache.spark.sql.catalyst.util.DateTimeUtils\nimport org.apache.spark.sql.types.DataTypes\nimport org.apache.spark.sql.types.DayTimeIntervalType\nimport org.apache.spark.sql.types.StructField\nimport org.apache.spark.sql.types.StructType\nimport org.apache.spark.sql.types.YearMonthIntervalType\nimport org.junit.Assert.assertArrayEquals\nimport org.junit.Assert.assertEquals\nimport org.junit.Assert.assertTrue\nimport org.junit.Assume\nimport org.junit.BeforeClass\nimport org.junit.Test\nimport org.neo4j.driver.types.IsoDuration\nimport org.neo4j.spark.util.ConstraintsOptimizationType\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.neo4j.spark.util.SchemaConstraintsOptimizationType\n\nimport java.sql.Date\nimport java.sql.Timestamp\nimport java.time.LocalDate\nimport java.time.LocalDateTime\nimport java.time.ZoneId\nimport java.time.ZoneOffset\nimport java.time.ZonedDateTime\nimport java.util.TimeZone\n\nimport scala.collection.JavaConverters.iterableAsScalaIterableConverter\nimport scala.collection.JavaConverters.mapAsScalaMapConverter\nimport scala.math.Ordering.Implicits.infixOrderingOps\n\nobject DataSourceSchemaWriterTSE {\n\n @BeforeClass\n def checkNeo4jVersion() {\n Assume.assumeTrue(TestUtil.neo4jVersion(SparkConnectorScalaSuiteIT.session()) >= Versions.NEO4J_5_13)\n }\n}\n\nclass DataSourceSchemaWriterTSE extends SparkConnectorScalaBaseTSE {\n val timeZoneLock = \"UTC\" // to make TIMESTAMP_NTZ tests deterministic\n\n final private val SHOW_CONSTRAINTS_QUERY =\n \"\"\"|SHOW CONSTRAINTS\n |YIELD name, type, entityType, labelsOrTypes, properties, ownedIndex, propertyType\"\"\".stripMargin\n\n final private val NODE_UNIQUENESS_SHOW_CONSTRAINTS_QUERY =\n \"\"\"|SHOW CONSTRAINTS\n |YIELD name, type AS ptype, entityType, labelsOrTypes, properties, ownedIndex, propertyType\n |RETURN name, entityType, labelsOrTypes, properties, ownedIndex,\n |CASE ptype\n | WHEN \"UNIQUENESS\" THEN \"NODE_PROPERTY_UNIQUENESS\"\n | ELSE ptype\n |END AS type, propertyType\n |ORDER BY type ASC\"\"\".stripMargin\n\n final private val RELATIONSHIP_UNIQUENESS_SHOW_CONSTRAINTS_QUERY =\n \"\"\"|SHOW CONSTRAINTS\n |YIELD name, type AS ptype, entityType, labelsOrTypes, properties, ownedIndex, propertyType\n |RETURN name, entityType, labelsOrTypes, properties, ownedIndex,\n |CASE ptype\n | WHEN \"RELATIONSHIP_UNIQUENESS\" THEN \"RELATIONSHIP_PROPERTY_UNIQUENESS\"\n | ELSE ptype\n |END AS type, propertyType\n |ORDER BY type ASC\"\"\".stripMargin\n\n final private val ALL_TYPES_AS_COL_NAMES = Array(\n \"string\",\n \"int\",\n \"boolean\",\n \"float\",\n \"date\",\n \"localDateTime\",\n \"zonedDateTime\",\n \"stringArray\",\n \"intArray\",\n \"booleanArray\",\n \"floatArray\",\n \"dateArray\",\n \"localDateTimeArray\",\n \"zonedDateTimeArray\"\n )\n\n val sparkSession = SparkSession.builder()\n .master(\"local[*]\")\n .appName(\"DataSourceWriterTSE\")\n .config(\"spark.sql.session.timeZone\", timeZoneLock) // to make TIMESTAMP_NTZ tests deterministic\n .getOrCreate()\n\n import sparkSession.implicits._\n\n private def mapData(data: Any): Any = data match {\n case null => null\n case a: Array[_] => a.toSeq.map(mapData)\n case l: java.util.List[_] => l.asScala.toSeq.map(mapData)\n case d: LocalDate => Date.valueOf(d)\n case zdt: ZonedDateTime => Timestamp.from(zdt.toInstant)\n case any: Any => any\n }\n\n private val schemaOptimization = SchemaConstraintsOptimizationType.values\n .filterNot(_ == SchemaConstraintsOptimizationType.NONE)\n .mkString(\",\")\n\n private val nodeWithSchema = \"NodeWithSchema\"\n\n @Test\n def shouldApplySchemaForNodes(): Unit = {\n val (expectedNode: Map[_root_.java.lang.String, Any], df: DataFrame) = createNodesDataFrameWithNotNullColumns\n\n df\n .write\n .mode(SaveMode.Append)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", s\":$nodeWithSchema\")\n .option(Neo4jOptions.SCHEMA_OPTIMIZATION, schemaOptimization)\n .save()\n\n val count: Long = SparkConnectorScalaSuiteIT.session().run(\n s\"\"\"\n |MATCH (n:$nodeWithSchema)\n |RETURN count(n)\n |\"\"\".stripMargin\n )\n .single()\n .get(0)\n .asLong()\n\n assertEquals(1L, count)\n\n val expectedSchema = Seq(\n constraintNodeNotNull(nodeWithSchema, \"boolean\"),\n constraintNodeNotNull(nodeWithSchema, \"float\"),\n constraintNodeNotNull(nodeWithSchema, \"int\"),\n constraintNodeNotNull(nodeWithSchema, \"string\"),\n constraintNodeType(nodeWithSchema, \"boolean\", \"BOOLEAN\"),\n constraintNodeType(nodeWithSchema, \"booleanArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"date\", \"DATE\"),\n constraintNodeType(nodeWithSchema, \"dateArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"float\", \"FLOAT\"),\n constraintNodeType(nodeWithSchema, \"floatArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"int\", \"INTEGER\"),\n constraintNodeType(nodeWithSchema, \"intArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"localDateTime\", \"LOCAL DATETIME\"),\n constraintNodeType(nodeWithSchema, \"localDateTimeArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"string\", \"STRING\"),\n constraintNodeType(nodeWithSchema, \"stringArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"zonedDateTime\", \"ZONED DATETIME\"),\n constraintNodeType(nodeWithSchema, \"zonedDateTimeArray\", \"LIST\")\n )\n\n val actualSchema = SparkConnectorScalaSuiteIT.session()\n .run(SHOW_CONSTRAINTS_QUERY)\n .list()\n .asScala\n .map(_.asMap(v => v.asObject()).asScala.mapValues(mapData).toMap)\n .toSeq\n\n assertEquals(expectedSchema, actualSchema)\n\n val actualNode = SparkConnectorScalaSuiteIT.session()\n .readTransaction(tx =>\n tx.run(s\"MATCH (n:$nodeWithSchema) RETURN n\")\n .list()\n .asScala\n .map(_.get(\"n\").asNode())\n .map(_.asMap())\n )\n .head\n .asScala\n .mapValues(mapData)\n .toMap\n\n assertEquals(expectedNode, actualNode)\n }\n\n @Test\n def shouldApplySchemaAndNodeKeysForNodes(): Unit = {\n val (expectedNode: Map[_root_.java.lang.String, Any], df: DataFrame) = createNodesDataFrameWithNotNullColumns\n\n df.write\n .mode(SaveMode.Overwrite)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", s\":$nodeWithSchema\")\n .option(Neo4jOptions.SCHEMA_OPTIMIZATION, schemaOptimization)\n .option(Neo4jOptions.SCHEMA_OPTIMIZATION_NODE_KEY, ConstraintsOptimizationType.KEY.toString)\n .option(\"node.keys\", \"int,string\")\n .save()\n\n val count: Long = SparkConnectorScalaSuiteIT.session().run(\n s\"\"\"\n |MATCH (n:$nodeWithSchema)\n |RETURN count(n)\n |\"\"\".stripMargin\n )\n .single()\n .get(0)\n .asLong()\n\n assertEquals(1L, count)\n\n val expectedSchema = Seq(\n constraintNodeNotNull(nodeWithSchema, \"boolean\"),\n constraintNodeNotNull(nodeWithSchema, \"float\"),\n constraintNodeNotNull(nodeWithSchema, \"int\"),\n constraintNodeNotNull(nodeWithSchema, \"string\"),\n constraintNodeType(nodeWithSchema, \"boolean\", \"BOOLEAN\"),\n constraintNodeType(nodeWithSchema, \"booleanArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"date\", \"DATE\"),\n constraintNodeType(nodeWithSchema, \"dateArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"float\", \"FLOAT\"),\n constraintNodeType(nodeWithSchema, \"floatArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"int\", \"INTEGER\"),\n constraintNodeType(nodeWithSchema, \"intArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"localDateTime\", \"LOCAL DATETIME\"),\n constraintNodeType(nodeWithSchema, \"localDateTimeArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"string\", \"STRING\"),\n constraintNodeType(nodeWithSchema, \"stringArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"zonedDateTime\", \"ZONED DATETIME\"),\n constraintNodeType(nodeWithSchema, \"zonedDateTimeArray\", \"LIST\"),\n constraintNodeKey(nodeWithSchema, Seq(\"int\", \"string\"))\n )\n\n val actualSchema = SparkConnectorScalaSuiteIT.session()\n .run(SHOW_CONSTRAINTS_QUERY)\n .list()\n .asScala\n .map(_.asMap(v => v.asObject()).asScala.mapValues(mapData).toMap)\n .toSeq\n\n assertEquals(expectedSchema, actualSchema)\n\n val actualNode = SparkConnectorScalaSuiteIT.session()\n .readTransaction(tx =>\n tx.run(s\"MATCH (n:$nodeWithSchema) RETURN n\")\n .list()\n .asScala\n .map(_.get(\"n\").asNode())\n .map(_.asMap())\n )\n .head\n .asScala\n .mapValues(mapData)\n .toMap\n\n assertEquals(expectedNode, actualNode)\n }\n\n @Test\n def shouldApplySchemaAndNodeKeysForNodesWhenRemapped(): Unit = {\n val (node: Map[_root_.java.lang.String, Any], df: DataFrame) = createNodesDataFrameWithNotNullColumns\n\n df.write\n .mode(SaveMode.Overwrite)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", s\":$nodeWithSchema\")\n .option(Neo4jOptions.SCHEMA_OPTIMIZATION, schemaOptimization)\n .option(Neo4jOptions.SCHEMA_OPTIMIZATION_NODE_KEY, ConstraintsOptimizationType.KEY.toString)\n .option(\"node.keys\", \"int:int_prop,string:string_prop\")\n .save()\n\n val count: Long = SparkConnectorScalaSuiteIT.session().run(\n s\"\"\"\n |MATCH (n:$nodeWithSchema)\n |RETURN count(n)\n |\"\"\".stripMargin\n )\n .single()\n .get(0)\n .asLong()\n\n assertEquals(1L, count)\n\n val expectedSchema = Seq(\n constraintNodeNotNull(nodeWithSchema, \"boolean\"),\n constraintNodeNotNull(nodeWithSchema, \"float\"),\n constraintNodeNotNull(nodeWithSchema, \"int_prop\"),\n constraintNodeNotNull(nodeWithSchema, \"string_prop\"),\n constraintNodeType(nodeWithSchema, \"boolean\", \"BOOLEAN\"),\n constraintNodeType(nodeWithSchema, \"booleanArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"date\", \"DATE\"),\n constraintNodeType(nodeWithSchema, \"dateArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"float\", \"FLOAT\"),\n constraintNodeType(nodeWithSchema, \"floatArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"intArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"int_prop\", \"INTEGER\"),\n constraintNodeType(nodeWithSchema, \"localDateTime\", \"LOCAL DATETIME\"),\n constraintNodeType(nodeWithSchema, \"localDateTimeArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"stringArray\", \"LIST\"),\n constraintNodeType(nodeWithSchema, \"string_prop\", \"STRING\"),\n constraintNodeType(nodeWithSchema, \"zonedDateTime\", \"ZONED DATETIME\"),\n constraintNodeType(nodeWithSchema, \"zonedDateTimeArray\", \"LIST\"),\n constraintNodeKey(nodeWithSchema, Seq(\"int_prop\", \"string_prop\"))\n )\n\n val actualSchema = SparkConnectorScalaSuiteIT.session()\n .run(SHOW_CONSTRAINTS_QUERY)\n .list()\n .asScala\n .map(_.asMap(v => v.asObject()).asScala.mapValues(mapData).toMap)\n .toSeq\n\n assertEquals(expectedSchema, actualSchema)\n\n val expectedNode =\n node.map {\n case (k, v) =>\n if (k == \"string\" || k == \"int\") (k + \"_prop\", v)\n else (k, v)\n }\n\n val actualNode = SparkConnectorScalaSuiteIT.session()\n .readTransaction(tx =>\n tx.run(s\"MATCH (n:$nodeWithSchema) RETURN n\")\n .list()\n .asScala\n .map(_.get(\"n\").asNode())\n .map(_.asMap())\n )\n .head\n .asScala\n .mapValues(mapData)\n .toMap\n\n assertEquals(expectedNode, actualNode)\n }\n\n final private def constraintNodeNotNull(node: String, prop: String): Map[String, Any] = Map(\n \"name\" -> s\"spark_NODE-NOT_NULL-CONSTRAINT-$node-$prop\",\n \"type\" -> \"NODE_PROPERTY_EXISTENCE\",\n \"entityType\" -> \"NODE\",\n \"labelsOrTypes\" -> Seq(node),\n \"properties\" -> Seq(prop),\n \"ownedIndex\" -> null,\n \"propertyType\" -> null\n )\n\n final private def constraintNodeType(node: String, prop: String, expectedType: String): Map[String, Any] = Map(\n \"name\" -> s\"spark_NODE-TYPE-CONSTRAINT-$node-$prop\",\n \"type\" -> \"NODE_PROPERTY_TYPE\",\n \"entityType\" -> \"NODE\",\n \"labelsOrTypes\" -> Seq(node),\n \"properties\" -> Seq(prop),\n \"ownedIndex\" -> null,\n \"propertyType\" -> expectedType\n )\n\n final private def constraintNodeKey(node: String, props: Seq[String]): Map[String, Any] = Map(\n \"name\" -> s\"spark_NODE_KEY-CONSTRAINT_${node}_${props.mkString(\"-\")}\",\n \"type\" -> \"NODE_KEY\",\n \"entityType\" -> \"NODE\",\n \"labelsOrTypes\" -> Seq(node),\n \"properties\" -> props,\n \"ownedIndex\" -> s\"spark_NODE_KEY-CONSTRAINT_${node}_${props.mkString(\"-\")}\",\n \"propertyType\" -> null\n )\n\n private def createNodesDataFrameWithNotNullColumns: (Map[String, Any], DataFrame) = {\n TimeZone.setDefault(TimeZone.getTimeZone(timeZoneLock))\n\n val row = (\n \"Foo\",\n 1,\n false,\n 1.1,\n Date.valueOf(\"2023-11-22\"),\n LocalDateTime.of(2023, 11, 22, 12, 12, 12),\n Timestamp.valueOf(s\"2020-11-22 11:11:11.11\"),\n Seq(\"Foo1\", \"Foo2\"),\n Seq(1, 2),\n Seq(true, false),\n Seq(1.1, 2.2),\n Seq(Date.valueOf(\"2023-11-22\"), Date.valueOf(\"2023-11-23\")),\n Seq(LocalDateTime.of(2023, 11, 22, 11, 11, 11), LocalDateTime.of(2023, 11, 23, 12, 12, 12)),\n Seq(Timestamp.valueOf(\"2023-11-22 11:11:11.11\"), Timestamp.valueOf(\"2023-11-23 12:12:12.12\"))\n )\n\n val data = Seq(row).toDF(ALL_TYPES_AS_COL_NAMES: _*)\n val expectedNode = ALL_TYPES_AS_COL_NAMES.zip(row.productIterator.toSeq).toMap\n\n val schema = StructType(data.schema.map { sf =>\n sf.name match {\n case \"localDateTimeArray\" =>\n StructField(sf.name, DataTypes.createArrayType(DataTypes.TimestampNTZType, false), sf.nullable)\n case \"zonedDateTimeArray\" =>\n StructField(sf.name, DataTypes.createArrayType(DataTypes.TimestampType, false), sf.nullable)\n case \"stringArray\" => StructField(sf.name, DataTypes.createArrayType(DataTypes.StringType, false), sf.nullable)\n case \"dateArray\" => StructField(sf.name, DataTypes.createArrayType(DataTypes.DateType, false), sf.nullable)\n case \"string\" => StructField(sf.name, DataTypes.StringType, false)\n case _ => sf\n }\n })\n\n val df = ss.createDataFrame(data.rdd, schema)\n (expectedNode, df)\n }\n\n @Test\n def shouldApplySchemaForRelationshipsAndNodes(): Unit = {\n val expectedMap = createDatasetForRelationships(\n Map(\n Neo4jOptions.SCHEMA_OPTIMIZATION -> schemaOptimization\n )\n )\n\n val count: Long = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH p = (:NodeA)-[:MY_REL]->(:NodeB)\n |RETURN count(p)\n |\"\"\".stripMargin\n )\n .single()\n .get(0)\n .asLong()\n\n assertEquals(1L, count)\n\n val expected = Seq(\n constraintNodeNotNull(\"NodeA\", \"id\"),\n constraintNodeNotNull(\"NodeB\", \"id\"),\n constraintNodeType(\"NodeA\", \"id\", \"STRING\"),\n constraintNodeType(\"NodeB\", \"id\", \"STRING\"),\n constraintRelNotNull(\"boolean\"),\n constraintRelNotNull(\"float\"),\n constraintRelNotNull(\"int\"),\n constraintRelType(\"boolean\", \"BOOLEAN\"),\n constraintRelType(\"booleanArray\", \"LIST\"),\n constraintRelType(\"date\", \"DATE\"),\n constraintRelType(\"dateArray\", \"LIST\"),\n constraintRelType(\"float\", \"FLOAT\"),\n constraintRelType(\"floatArray\", \"LIST\"),\n constraintRelType(\"int\", \"INTEGER\"),\n constraintRelType(\"intArray\", \"LIST\"),\n constraintRelType(\"localDateTime\", \"LOCAL DATETIME\"),\n constraintRelType(\"localDateTimeArray\", \"LIST\"),\n constraintRelType(\"string\", \"STRING\"),\n constraintRelType(\"stringArray\", \"LIST\"),\n constraintRelType(\"zonedDateTime\", \"ZONED DATETIME\"),\n constraintRelType(\"zonedDateTimeArray\", \"LIST\")\n )\n\n val actual = SparkConnectorScalaSuiteIT.session()\n .run(SHOW_CONSTRAINTS_QUERY)\n .list()\n .asScala\n .map(_.asMap(v => v.asObject()).asScala.mapValues(mapData).toMap)\n .toSeq\n\n assertEquals(expected, actual)\n\n val actualMap = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (s:NodeA)-[r:MY_REL]->(t:NodeB)\n |RETURN s.id AS idSource, t.id AS idTarget, r\n |\"\"\".stripMargin\n )\n .list()\n .asScala\n .map(r =>\n Map(\"idSource\" -> r.get(\"idSource\").asString(), \"idTarget\" -> r.get(\"idTarget\").asString()) ++ r.get(\n \"r\"\n ).asRelationship().asMap().asScala\n )\n .head\n .mapValues(mapData)\n .toMap\n\n assertEquals(expectedMap, actualMap)\n }\n\n private def createDatasetForRelationships(options: Map[String, String]): Map[String, Any] = {\n val shouldRemap = options.contains(Neo4jOptions.RELATIONSHIP_PROPERTIES)\n\n SparkConnectorScalaSuiteIT.session()\n .run(\"CREATE (:NodeA{id: 'a'}), (:NodeB{id: 'b'})\")\n .consume()\n\n val colNames = Array(\n \"idSource\",\n \"idTarget\"\n ) ++ ALL_TYPES_AS_COL_NAMES\n\n val row = (\n \"a\",\n \"b\",\n \"Foo\",\n 1,\n false,\n 1.1,\n Date.valueOf(\"2023-11-22\"),\n LocalDateTime.of(2023, 11, 22, 12, 12, 12),\n Timestamp.valueOf(s\"2020-11-22 11:11:11.11\"),\n Seq(\"Foo1\", \"Foo2\"),\n Seq(1, 2),\n Seq(true, false),\n Seq(1.1, 2.2),\n Seq(Date.valueOf(\"2023-11-22\"), Date.valueOf(\"2023-11-23\")),\n Seq(LocalDateTime.of(2023, 11, 22, 11, 11, 11), LocalDateTime.of(2023, 11, 23, 12, 12, 12)),\n Seq(Timestamp.valueOf(\"2023-11-22 11:11:11.11\"), Timestamp.valueOf(\"2023-11-23 12:12:12.12\"))\n )\n\n val data = Seq(row).toDF(colNames: _*)\n\n val schema = StructType(data.schema.map { sf =>\n sf.name match {\n case \"localDateTimeArray\" =>\n StructField(sf.name, DataTypes.createArrayType(DataTypes.TimestampNTZType, false), sf.nullable)\n case \"zonedDateTimeArray\" =>\n StructField(sf.name, DataTypes.createArrayType(DataTypes.TimestampType, false), sf.nullable)\n case \"stringArray\" => StructField(sf.name, DataTypes.createArrayType(DataTypes.StringType, false), sf.nullable)\n case \"dateArray\" => StructField(sf.name, DataTypes.createArrayType(DataTypes.DateType, false), sf.nullable)\n case \"idSource\" => StructField(sf.name, DataTypes.StringType, false)\n case \"idTarget\" => StructField(sf.name, DataTypes.StringType, false)\n case _ => sf\n }\n })\n\n ss.createDataFrame(data.rdd, schema)\n .write\n .mode(SaveMode.Overwrite)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"MY_REL\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":NodeA\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.source.node.keys\", \"idSource:id\")\n .option(\"relationship.target.labels\", \":NodeB\")\n .option(\"relationship.target.node.keys\", \"idTarget:id\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .options(options)\n .save()\n\n colNames.map(c =>\n if (shouldRemap && (c == \"string\" || c == \"int\")) c + \"_prop\"\n else c\n ).zip(row.productIterator.toSeq).toMap\n }\n\n final private def constraintRelNotNull(prop: String): Map[String, Any] = Map(\n \"name\" -> s\"spark_RELATIONSHIP-NOT_NULL-CONSTRAINT-MY_REL-$prop\",\n \"type\" -> \"RELATIONSHIP_PROPERTY_EXISTENCE\",\n \"entityType\" -> \"RELATIONSHIP\",\n \"labelsOrTypes\" -> Seq(\"MY_REL\"),\n \"properties\" -> Seq(prop),\n \"ownedIndex\" -> null,\n \"propertyType\" -> null\n )\n\n final private def constraintRelType(prop: String, expectedType: String) = Map(\n \"name\" -> s\"spark_RELATIONSHIP-TYPE-CONSTRAINT-MY_REL-$prop\",\n \"type\" -> \"RELATIONSHIP_PROPERTY_TYPE\",\n \"entityType\" -> \"RELATIONSHIP\",\n \"labelsOrTypes\" -> Seq(\"MY_REL\"),\n \"properties\" -> Seq(prop),\n \"ownedIndex\" -> null,\n \"propertyType\" -> expectedType\n )\n\n @Test\n def shouldApplySchemaForRelationshipsAndNodesWhenRemapped(): Unit = {\n val expectedMap = createDatasetForRelationships(\n Map(\n Neo4jOptions.SCHEMA_OPTIMIZATION -> schemaOptimization,\n Neo4jOptions.RELATIONSHIP_PROPERTIES -> ALL_TYPES_AS_COL_NAMES.map {\n case \"string\" => \"string:string_prop\"\n case \"int\" => \"int:int_prop\"\n case c => c\n }.mkString(\",\")\n )\n )\n\n val count: Long = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH p = (:NodeA)-[:MY_REL]->(:NodeB)\n |RETURN count(p)\n |\"\"\".stripMargin\n )\n .single()\n .get(0)\n .asLong()\n\n assertEquals(1L, count)\n\n val expected = Seq(\n constraintNodeNotNull(\"NodeA\", \"id\"),\n constraintNodeNotNull(\"NodeB\", \"id\"),\n constraintNodeType(\"NodeA\", \"id\", \"STRING\"),\n constraintNodeType(\"NodeB\", \"id\", \"STRING\"),\n constraintRelNotNull(\"boolean\"),\n constraintRelNotNull(\"float\"),\n constraintRelNotNull(\"int_prop\"),\n constraintRelType(\"boolean\", \"BOOLEAN\"),\n constraintRelType(\"booleanArray\", \"LIST\"),\n constraintRelType(\"date\", \"DATE\"),\n constraintRelType(\"dateArray\", \"LIST\"),\n constraintRelType(\"float\", \"FLOAT\"),\n constraintRelType(\"floatArray\", \"LIST\"),\n constraintRelType(\"intArray\", \"LIST\"),\n constraintRelType(\"int_prop\", \"INTEGER\"),\n constraintRelType(\"localDateTime\", \"LOCAL DATETIME\"),\n constraintRelType(\"localDateTimeArray\", \"LIST\"),\n constraintRelType(\"stringArray\", \"LIST\"),\n constraintRelType(\"string_prop\", \"STRING\"),\n constraintRelType(\"zonedDateTime\", \"ZONED DATETIME\"),\n constraintRelType(\"zonedDateTimeArray\", \"LIST\")\n )\n\n val actual = SparkConnectorScalaSuiteIT.session()\n .run(SHOW_CONSTRAINTS_QUERY)\n .list()\n .asScala\n .map(_.asMap(v => v.asObject()).asScala.mapValues(mapData).toMap)\n .toSeq\n\n assertEquals(expected, actual)\n\n val actualMap = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (s:NodeA)-[r:MY_REL]->(t:NodeB)\n |RETURN s.id AS idSource, t.id AS idTarget, r\n |\"\"\".stripMargin\n )\n .list()\n .asScala\n .map(r =>\n Map(\"idSource\" -> r.get(\"idSource\").asString(), \"idTarget\" -> r.get(\"idTarget\").asString()) ++ r.get(\n \"r\"\n ).asRelationship().asMap().asScala\n )\n .head\n .mapValues(mapData)\n .toMap\n\n assertEquals(expectedMap, actualMap)\n }\n\n @Test\n def shouldApplyUniqueConstraintForNode(): Unit = {\n val total = 10\n val ds = (1 to total)\n .map(i => i.toString)\n .toDF(\"surname\")\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":Person:Customer\")\n .option(\"node.keys\", \"surname\")\n .option(Neo4jOptions.SCHEMA_OPTIMIZATION_NODE_KEY, ConstraintsOptimizationType.UNIQUE.toString)\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:Person:Customer)\n |RETURN p.surname AS surname\n |\"\"\".stripMargin\n ).list().asScala\n .map(r => r.asMap().asScala)\n .toSet\n val expected = ds.collect().map(row => Map(\"surname\" -> row.getAs[String](\"surname\")))\n .toSet\n assertEquals(expected, records)\n\n val actualConstraint = SparkConnectorScalaSuiteIT.session().run(NODE_UNIQUENESS_SHOW_CONSTRAINTS_QUERY)\n .list()\n .asScala\n .map(_.asMap(v => v.asObject()).asScala.mapValues(mapData).toMap)\n .head\n val expectedConstraint = Map(\n \"name\" -> \"spark_NODE_UNIQUE-CONSTRAINT_Person_surname\",\n \"type\" -> \"NODE_PROPERTY_UNIQUENESS\",\n \"entityType\" -> \"NODE\",\n \"labelsOrTypes\" -> Seq(\"Person\"),\n \"properties\" -> Seq(\"surname\"),\n \"ownedIndex\" -> \"spark_NODE_UNIQUE-CONSTRAINT_Person_surname\",\n \"propertyType\" -> null\n )\n assertEquals(expectedConstraint, actualConstraint)\n\n SparkConnectorScalaSuiteIT.session().run(\"DROP CONSTRAINT `spark_NODE_UNIQUE-CONSTRAINT_Person_surname`\").consume()\n }\n\n @Test\n def shouldApplyNodeKeyConstraintForNode(): Unit = {\n val total = 10\n val ds = (1 to total)\n .map(i => i.toString)\n .toDF(\"surname\")\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":Person:Customer\")\n .option(\"node.keys\", \"surname\")\n .option(Neo4jOptions.SCHEMA_OPTIMIZATION_NODE_KEY, ConstraintsOptimizationType.KEY.toString)\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:Person:Customer)\n |RETURN p.surname AS surname\n |\"\"\".stripMargin\n ).list().asScala\n .map(r => r.asMap().asScala)\n .toSet\n val expected = ds.collect().map(row => Map(\"surname\" -> row.getAs[String](\"surname\")))\n .toSet\n assertEquals(expected, records)\n\n val actualConstraint = SparkConnectorScalaSuiteIT.session().run(SHOW_CONSTRAINTS_QUERY)\n .list()\n .asScala\n .map(_.asMap(v => v.asObject()).asScala.mapValues(mapData).toMap)\n .head\n val expectedConstraint = Map(\n \"name\" -> \"spark_NODE_KEY-CONSTRAINT_Person_surname\",\n \"type\" -> \"NODE_KEY\",\n \"entityType\" -> \"NODE\",\n \"labelsOrTypes\" -> Seq(\"Person\"),\n \"properties\" -> Seq(\"surname\"),\n \"ownedIndex\" -> \"spark_NODE_KEY-CONSTRAINT_Person_surname\",\n \"propertyType\" -> null\n )\n assertEquals(expectedConstraint, actualConstraint)\n\n SparkConnectorScalaSuiteIT.session().run(\"DROP CONSTRAINT `spark_NODE_KEY-CONSTRAINT_Person_surname`\").consume()\n }\n\n @Test\n def shouldApplyAppropriateConstraintsEvenWhenRemapped(): Unit = {\n val total = 10\n val ds = (1 to total)\n .map(i => i.toString)\n .toDF(\"surname\")\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":SurnameKey\")\n .option(\"node.keys\", \"surname:surname_key\")\n .option(Neo4jOptions.SCHEMA_OPTIMIZATION_NODE_KEY, ConstraintsOptimizationType.KEY.toString)\n .save()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":SurnameUnique\")\n .option(\"node.keys\", \"surname:surname_unique\")\n .option(Neo4jOptions.SCHEMA_OPTIMIZATION_NODE_KEY, ConstraintsOptimizationType.UNIQUE.toString)\n .save()\n\n val actualConstraint = SparkConnectorScalaSuiteIT.session().run(NODE_UNIQUENESS_SHOW_CONSTRAINTS_QUERY)\n .list()\n .asScala\n .map(_.asMap(v => v.asObject()).asScala.mapValues(mapData).toMap)\n .toSeq\n\n val expectedConstraint = Seq(\n Map(\n \"name\" -> \"spark_NODE_KEY-CONSTRAINT_SurnameKey_surname_key\",\n \"type\" -> \"NODE_KEY\",\n \"entityType\" -> \"NODE\",\n \"labelsOrTypes\" -> Seq(\"SurnameKey\"),\n \"properties\" -> Seq(\"surname_key\"),\n \"ownedIndex\" -> \"spark_NODE_KEY-CONSTRAINT_SurnameKey_surname_key\",\n \"propertyType\" -> null\n ),\n Map(\n \"name\" -> \"spark_NODE_UNIQUE-CONSTRAINT_SurnameUnique_surname_unique\",\n \"type\" -> \"NODE_PROPERTY_UNIQUENESS\",\n \"entityType\" -> \"NODE\",\n \"labelsOrTypes\" -> Seq(\"SurnameUnique\"),\n \"properties\" -> Seq(\"surname_unique\"),\n \"ownedIndex\" -> \"spark_NODE_UNIQUE-CONSTRAINT_SurnameUnique_surname_unique\",\n \"propertyType\" -> null\n )\n )\n\n assertEquals(expectedConstraint, actualConstraint)\n\n SparkConnectorScalaSuiteIT.session().run(\n \"DROP CONSTRAINT `spark_NODE_KEY-CONSTRAINT_SurnameKey_surname_key`\"\n ).consume()\n SparkConnectorScalaSuiteIT.session().run(\n \"DROP CONSTRAINT `spark_NODE_UNIQUE-CONSTRAINT_SurnameUnique_surname_unique`\"\n ).consume()\n }\n\n @Test\n def shouldApplyUniqueConstraintForRelationship(): Unit = {\n val expectedMap = createDatasetForRelationships(\n Map(\n Neo4jOptions.SCHEMA_OPTIMIZATION_RELATIONSHIP_KEY -> ConstraintsOptimizationType.UNIQUE.toString,\n \"relationship.keys\" -> \"string,int\"\n )\n )\n val actualMap = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (s:NodeA)-[r:MY_REL]->(t:NodeB)\n |RETURN s.id AS idSource, t.id AS idTarget, r\n |\"\"\".stripMargin\n )\n .list()\n .asScala\n .map(r =>\n Map(\"idSource\" -> r.get(\"idSource\").asString(), \"idTarget\" -> r.get(\"idTarget\").asString()) ++ r.get(\n \"r\"\n ).asRelationship().asMap().asScala\n )\n .head\n .mapValues(mapData)\n .toMap\n\n assertEquals(expectedMap, actualMap)\n\n val actualConstraint = SparkConnectorScalaSuiteIT.session().run(RELATIONSHIP_UNIQUENESS_SHOW_CONSTRAINTS_QUERY)\n .list()\n .asScala\n .map(_.asMap(v => v.asObject()).asScala.mapValues(mapData).toMap)\n .head\n val expectedConstraint = Map(\n \"name\" -> \"spark_RELATIONSHIP_UNIQUE-CONSTRAINT_MY_REL_string-int\",\n \"type\" -> \"RELATIONSHIP_PROPERTY_UNIQUENESS\",\n \"entityType\" -> \"RELATIONSHIP\",\n \"labelsOrTypes\" -> Seq(\"MY_REL\"),\n \"properties\" -> Seq(\"string\", \"int\"),\n \"ownedIndex\" -> \"spark_RELATIONSHIP_UNIQUE-CONSTRAINT_MY_REL_string-int\",\n \"propertyType\" -> null\n )\n assertEquals(expectedConstraint, actualConstraint)\n }\n\n @Test\n def shouldApplyRelUniqueConstraintForRelationship(): Unit = {\n val expectedMap = createDatasetForRelationships(\n Map(\n Neo4jOptions.SCHEMA_OPTIMIZATION_RELATIONSHIP_KEY -> ConstraintsOptimizationType.KEY.toString,\n \"relationship.keys\" -> \"string,int\"\n )\n )\n\n val actualMap = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (s:NodeA)-[r:MY_REL]->(t:NodeB)\n |RETURN s.id AS idSource, t.id AS idTarget, r\n |\"\"\".stripMargin\n )\n .list()\n .asScala\n .map(r =>\n Map(\"idSource\" -> r.get(\"idSource\").asString(), \"idTarget\" -> r.get(\"idTarget\").asString()) ++ r.get(\n \"r\"\n ).asRelationship().asMap().asScala\n )\n .head\n .mapValues(mapData)\n .toMap\n\n assertEquals(expectedMap, actualMap)\n\n val actualConstraint = SparkConnectorScalaSuiteIT.session().run(SHOW_CONSTRAINTS_QUERY)\n .list()\n .asScala\n .map(_.asMap(v => v.asObject()).asScala.mapValues(mapData).toMap)\n .head\n val expectedConstraint = Map(\n \"name\" -> \"spark_RELATIONSHIP_KEY-CONSTRAINT_MY_REL_string-int\",\n \"type\" -> \"RELATIONSHIP_KEY\",\n \"entityType\" -> \"RELATIONSHIP\",\n \"labelsOrTypes\" -> Seq(\"MY_REL\"),\n \"properties\" -> Seq(\"string\", \"int\"),\n \"ownedIndex\" -> \"spark_RELATIONSHIP_KEY-CONSTRAINT_MY_REL_string-int\",\n \"propertyType\" -> null\n )\n\n assertEquals(expectedConstraint, actualConstraint)\n }\n\n @Test\n def shouldWriteNodeWithLegacyTypeConversionDisabledByDefault(): Unit = {\n val df = sparkSession.sql(\n \"\"\"\n |SELECT\n | 'legacy-type-conversion' AS id,\n | timestamp('2025-01-01 11:11:11') AS timestamp,\n | CAST('2025-01-01 11:11:11' AS TIMESTAMP_NTZ) AS timestampNtz,\n | INTERVAL '4' DAY AS dayInterval,\n | INTERVAL '10 05' DAY TO HOUR AS dayToHour,\n | timestamp('2025-01-02 18:30:00.454') - timestamp('2024-01-01 00:00:00') AS arithmeticDuration,\n | INTERVAL '3' YEAR AS yearInterval,\n | INTERVAL '1-2' YEAR TO MONTH AS yearToMonth,\n | CAST('erik' AS BINARY) AS binary,\n | CAST(array(1, 2, 3) AS array) AS byteArray\n |\"\"\".stripMargin\n )\n assertTrue(df.schema(\"dayInterval\").dataType.isInstanceOf[DayTimeIntervalType])\n assertTrue(df.schema(\"dayToHour\").dataType.isInstanceOf[DayTimeIntervalType])\n assertTrue(df.schema(\"arithmeticDuration\").dataType.isInstanceOf[DayTimeIntervalType])\n assertTrue(df.schema(\"yearInterval\").dataType.isInstanceOf[YearMonthIntervalType])\n assertTrue(df.schema(\"yearToMonth\").dataType.isInstanceOf[YearMonthIntervalType])\n\n df.write\n .mode(SaveMode.Overwrite)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":LegacyTypeConversionDisabled\")\n .option(\"node.keys\", \"id\")\n .save()\n\n val actual = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (n:LegacyTypeConversionDisabled {id: 'legacy-type-conversion'})\n |RETURN n\n |\"\"\".stripMargin\n )\n .single()\n .get(\"n\")\n .asNode()\n .asMap()\n .asScala\n .toMap\n assertEquals(\n Set(\n \"id\",\n \"timestamp\",\n \"timestampNtz\",\n \"dayInterval\",\n \"dayToHour\",\n \"arithmeticDuration\",\n \"yearInterval\",\n \"yearToMonth\",\n \"binary\",\n \"byteArray\"\n ),\n actual.keySet\n )\n assertEquals(\"legacy-type-conversion\", actual(\"id\"))\n val expectedZoned = ZonedDateTime.of(2025, 1, 1, 11, 11, 11, 0, ZoneOffset.UTC)\n assertEquals(expectedZoned, actual(\"timestamp\"))\n val expectedNtz = LocalDateTime.of(2025, 1, 1, 11, 11, 11)\n assertEquals(expectedNtz, actual(\"timestampNtz\"))\n val dayInterval = actual(\"dayInterval\").asInstanceOf[IsoDuration]\n assertEquals(0L, dayInterval.months())\n assertEquals(4L, dayInterval.days())\n assertEquals(0L, dayInterval.seconds())\n assertEquals(0, dayInterval.nanoseconds())\n val dayToHour = actual(\"dayToHour\").asInstanceOf[IsoDuration]\n assertEquals(0L, dayToHour.months())\n assertEquals(10L, dayToHour.days())\n assertEquals(18000L, dayToHour.seconds())\n assertEquals(0, dayToHour.nanoseconds())\n val arithmetic = actual(\"arithmeticDuration\").asInstanceOf[IsoDuration]\n assertEquals(0L, arithmetic.months())\n assertEquals(367L, arithmetic.days())\n assertEquals(66600L, arithmetic.seconds())\n assertEquals(454000000, arithmetic.nanoseconds())\n val yearInterval = actual(\"yearInterval\").asInstanceOf[IsoDuration]\n assertEquals(36L, yearInterval.months())\n assertEquals(0L, yearInterval.days())\n assertEquals(0L, yearInterval.seconds())\n assertEquals(0, yearInterval.nanoseconds())\n val yearToMonth = actual(\"yearToMonth\").asInstanceOf[IsoDuration]\n assertEquals(14L, yearToMonth.months())\n assertEquals(0L, yearToMonth.days())\n assertEquals(0L, yearToMonth.seconds())\n assertEquals(0, yearToMonth.nanoseconds())\n assertArrayEquals(Array[Byte](101, 114, 105, 107), actual(\"binary\").asInstanceOf[Array[Byte]])\n assertArrayEquals(Array[Byte](1, 2, 3), actual(\"byteArray\").asInstanceOf[Array[Byte]])\n }\n\n @Test\n def shouldWriteNodeWithLegacyTypeConversionEnabled(): Unit = {\n val df = sparkSession.sql(\n \"\"\"\n |SELECT\n | 'legacy-type-conversion' AS id,\n | timestamp('2025-01-01 11:11:11') AS timestamp,\n | CAST('2025-01-01 11:11:11' AS TIMESTAMP_NTZ) AS timestampNtz,\n | INTERVAL '4' DAY AS dayInterval,\n | INTERVAL '10 05' DAY TO HOUR AS dayToHour,\n | timestamp('2025-01-02 18:30:00.454') - timestamp('2024-01-01 00:00:00') AS arithmeticDuration,\n | INTERVAL '3' YEAR AS yearInterval,\n | INTERVAL '1-2' YEAR TO MONTH AS yearToMonth,\n | CAST('erik' AS BINARY) AS binary,\n | CAST(array(1, 2, 3) AS array) AS byteArray\n |\"\"\".stripMargin\n )\n assertTrue(df.schema(\"dayInterval\").dataType.isInstanceOf[DayTimeIntervalType])\n assertTrue(df.schema(\"dayToHour\").dataType.isInstanceOf[DayTimeIntervalType])\n assertTrue(df.schema(\"arithmeticDuration\").dataType.isInstanceOf[DayTimeIntervalType])\n assertTrue(df.schema(\"yearInterval\").dataType.isInstanceOf[YearMonthIntervalType])\n assertTrue(df.schema(\"yearToMonth\").dataType.isInstanceOf[YearMonthIntervalType])\n\n df.write\n .mode(SaveMode.Overwrite)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":LegacyTypeConversionEnabled\")\n .option(\"node.keys\", \"id\")\n .option(Neo4jOptions.TYPE_CONVERSION, \"legacy\")\n .save()\n\n val actual = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (n:LegacyTypeConversionEnabled {id: 'legacy-type-conversion'})\n |RETURN n\n |\"\"\".stripMargin\n )\n .single()\n .get(\"n\")\n .asNode()\n .asMap()\n .asScala\n .toMap\n assertEquals(\n Set(\n \"id\",\n \"timestamp\",\n \"timestampNtz\",\n \"dayInterval\",\n \"dayToHour\",\n \"arithmeticDuration\",\n \"yearInterval\",\n \"yearToMonth\",\n \"binary\",\n \"byteArray\"\n ),\n actual.keySet\n )\n assertEquals(\"legacy-type-conversion\", actual(\"id\"))\n val expectedTimestamp = ZonedDateTime.of(2025, 1, 1, 11, 11, 11, 0, ZoneOffset.UTC)\n .withZoneSameInstant(ZoneId.systemDefault())\n .toLocalDateTime\n assertEquals(expectedTimestamp, actual(\"timestamp\"))\n assertEquals(\n DateTimeUtils.localDateTimeToMicros(LocalDateTime.of(2025, 1, 1, 11, 11, 11)),\n actual(\"timestampNtz\").asInstanceOf[java.lang.Number].longValue()\n )\n assertEquals(4L * 24L * 3600L * 1000000L, actual(\"dayInterval\").asInstanceOf[java.lang.Number].longValue())\n assertEquals((10L * 24L + 5L) * 3600L * 1000000L, actual(\"dayToHour\").asInstanceOf[java.lang.Number].longValue())\n assertEquals(\n (367L * 24L * 3600L + 66600L) * 1000000L + 454000L,\n actual(\"arithmeticDuration\").asInstanceOf[java.lang.Number].longValue()\n )\n assertEquals(36L, actual(\"yearInterval\").asInstanceOf[java.lang.Number].longValue())\n assertEquals(14L, actual(\"yearToMonth\").asInstanceOf[java.lang.Number].longValue())\n assertArrayEquals(Array[Byte](101, 114, 105, 107), actual(\"binary\").asInstanceOf[Array[Byte]])\n val byteArray = actual(\"byteArray\").asInstanceOf[java.util.List[_]].asScala\n .map(_.asInstanceOf[java.lang.Number].longValue())\n assertEquals(Seq(1L, 2L, 3L), byteArray)\n }\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DataSourceStreamingReaderTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.sql.Row\nimport org.apache.spark.sql.streaming.StreamingQuery\nimport org.apache.spark.sql.streaming.Trigger\nimport org.hamcrest.Matchers\nimport org.junit.After\nimport org.junit.Assert.assertEquals\nimport org.junit.Assert.assertTrue\nimport org.junit.Rule\nimport org.junit.Test\nimport org.junit.rules.TemporaryFolder\nimport org.neo4j.Closeables.use\nimport org.neo4j.spark.SparkConnectorScalaSuiteIT.session\n\nimport java.util.concurrent.Executors\nimport java.util.concurrent.TimeUnit\n\nimport scala.annotation.meta.getter\n\nclass DataSourceStreamingReaderTSE extends SparkConnectorScalaBaseTSE {\n\n @(Rule @getter)\n val folder: TemporaryFolder = new TemporaryFolder()\n\n private var query: StreamingQuery = _\n\n @After\n def close(): Unit = {\n if (query != null) {\n query.stop()\n }\n }\n\n @Test\n def testReadStreamWithLabels(): Unit = {\n createMovieNodes(0, 1)\n\n val stream = ss.readStream.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Movie\")\n .option(\"streaming.property.name\", \"timestamp\")\n .option(\"streaming.from\", \"NOW\")\n .load()\n\n query = stream.writeStream\n .format(\"memory\")\n .queryName(\"readStreamWithLabels\")\n .start()\n\n val total = 60\n val expected: Seq[Map[String, Any]] = (1 to total).map(index =>\n Map(\n \"\" -> Seq(\"Movie\"),\n \"title\" -> s\"My movie $index\"\n )\n )\n\n // Continue creating nodes in the background\n Executors.newSingleThreadExecutor().submit(new Runnable {\n override def run(): Unit = {\n createMovieNodes(1, total, 1000, 200)\n }\n })\n\n Assert.assertEventually(\n new Assert.ThrowingSupplier[Seq[Map[String, Any]], Exception] {\n override def get(): Seq[Map[String, Any]] = {\n selectRowsFromTable(\"select * from readStreamWithLabels order by timestamp\", mapMovie)\n }\n },\n Matchers.equalTo(expected),\n 30L,\n TimeUnit.SECONDS\n )\n }\n\n @Test\n def testReadStreamWithLabelsGetAll(): Unit = {\n createMovieNodes(0, 1)\n\n val stream = ss.readStream.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Movie\")\n .option(\"streaming.property.name\", \"timestamp\")\n .option(\"streaming.from\", \"ALL\")\n .load()\n\n query = stream.writeStream\n .format(\"memory\")\n .queryName(\"readStreamWithLabelsAll\")\n .start()\n\n val total = 60\n Executors.newSingleThreadExecutor().submit(new Runnable {\n override def run(): Unit = {\n createMovieNodes(1, total, 1000, 200)\n }\n })\n\n val expected: Seq[Map[String, Any]] = (0 to total).map(index =>\n Map(\n \"\" -> Seq(\"Movie\"),\n \"title\" -> s\"My movie $index\"\n )\n )\n\n Assert.assertEventually(\n new Assert.ThrowingSupplier[Seq[Map[String, Any]], Exception] {\n override def get(): Seq[Map[String, Any]] = {\n selectRowsFromTable(\"select * from readStreamWithLabelsAll order by timestamp\", mapMovie)\n }\n },\n Matchers.equalTo(expected),\n 30L,\n TimeUnit.SECONDS\n )\n }\n\n @Test\n def testReadStreamWithLabelsResumesFromCheckpoint(): Unit = {\n createMovieNodes(0, 1)\n\n val stream = ss.readStream.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Movie\")\n .option(\"streaming.property.name\", \"timestamp\")\n .option(\"streaming.from\", \"NOW\")\n .load()\n\n val total = 60\n val expected: Seq[Map[String, Any]] = (1 to total).map(index =>\n Map(\n \"\" -> List(\"Movie\"),\n \"title\" -> s\"My movie $index\"\n )\n )\n\n val checkpoint = folder.newFolder()\n\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithLabelsCheckpoint\")\n .awaitTermination()\n\n val partial: Int = total / 2\n // create partial movies starting from 1\n createMovieNodes(1, partial, 0, 10)\n\n // fetch whatever is available\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithLabelsCheckpoint\")\n .awaitTermination()\n\n // create rest of the movies starting from partial+1\n createMovieNodes(partial + 1, total - partial, 0, 10)\n\n // fetch rest of the items from where we left off\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithLabelsCheckpoint\")\n .awaitTermination()\n\n assertEquals(\n expected,\n selectRowsFromTable(\"select * from readStreamWithLabelsCheckpoint order by timestamp\", mapMovie)\n )\n }\n\n @Test\n def testReadStreamWithRelationship(): Unit = {\n createLikesRelationships(0, 1)\n\n val stream = ss.readStream.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"LIKES\")\n .option(\"streaming.property.name\", \"timestamp\")\n .option(\"streaming.from\", \"NOW\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Post\")\n .load()\n\n query = stream.writeStream\n .format(\"memory\")\n .queryName(\"readStreamWithRelationship\")\n .start()\n\n val total = 60\n\n val expected: Seq[Map[String, Any]] = (1 to total).map(index =>\n Map(\n \"\" -> \"LIKES\",\n \"\" -> Seq(\"Person\"),\n \"source.age\" -> index,\n \"\" -> Seq(\"Post\"),\n \"target.hash\" -> s\"hash$index\",\n \"rel.id\" -> index\n )\n )\n\n Executors.newSingleThreadExecutor().submit(new Runnable {\n override def run(): Unit = {\n createLikesRelationships(1, total, 1000, 200)\n }\n })\n\n Assert.assertEventually(\n new Assert.ThrowingSupplier[Seq[Map[String, Any]], Exception] {\n override def get(): Seq[Map[String, Any]] = {\n selectRowsFromTable(\"select * from readStreamWithRelationship order by `rel.timestamp`\", mapLikes)\n }\n },\n Matchers.equalTo(expected),\n 30L,\n TimeUnit.SECONDS\n )\n }\n\n @Test\n def testReadStreamWithRelationshipGetAll(): Unit = {\n createLikesRelationships(0, 1)\n\n val stream = ss.readStream.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"LIKES\")\n .option(\"streaming.property.name\", \"timestamp\")\n .option(\"streaming.from\", \"ALL\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Post\")\n .load()\n\n query = stream.writeStream\n .format(\"memory\")\n .queryName(\"readStreamWithRelationshipAll\")\n .start()\n\n val total = 60\n val expected: Seq[Map[String, Any]] = (0 to total).map(index =>\n Map(\n \"\" -> \"LIKES\",\n \"\" -> Seq(\"Person\"),\n \"source.age\" -> index,\n \"\" -> Seq(\"Post\"),\n \"target.hash\" -> s\"hash$index\",\n \"rel.id\" -> index\n )\n )\n\n Executors.newSingleThreadExecutor().submit(new Runnable {\n override def run(): Unit = {\n createLikesRelationships(1, total, 1000, 200)\n }\n })\n\n Assert.assertEventually(\n new Assert.ThrowingSupplier[Seq[Map[String, Any]], Exception] {\n override def get(): Seq[Map[String, Any]] = {\n selectRowsFromTable(\"select * from readStreamWithRelationshipAll order by `rel.timestamp`\", mapLikes)\n }\n },\n Matchers.equalTo(expected),\n 30L,\n TimeUnit.SECONDS\n )\n }\n\n @Test\n def testReadStreamWithRelationshipResumesFromCheckpoint(): Unit = {\n createLikesRelationships(0, 1)\n\n val stream = ss.readStream.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"LIKES\")\n .option(\"streaming.property.name\", \"timestamp\")\n .option(\"streaming.from\", \"ALL\")\n .option(\"relationship.source.labels\", \"Person\")\n .option(\"relationship.target.labels\", \"Post\")\n .load()\n\n val total = 60\n val expected: Seq[Map[String, Any]] = (0 to total).map(index =>\n Map(\n \"\" -> \"LIKES\",\n \"\" -> Seq(\"Person\"),\n \"source.age\" -> index,\n \"\" -> Seq(\"Post\"),\n \"target.hash\" -> s\"hash$index\",\n \"rel.id\" -> index\n )\n )\n\n val checkpoint = folder.newFolder()\n\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithRelationshipCheckpoint\")\n .awaitTermination()\n\n val partial: Int = total / 2\n // create partial number of likes starting from 1\n createLikesRelationships(1, partial, 0, 10)\n\n // fetch whatever is available\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithRelationshipCheckpoint\")\n .awaitTermination()\n\n // create rest of the likes starting from partial+1\n createLikesRelationships(partial + 1, total - partial, 0, 10)\n\n // fetch rest of the items from where we left off\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithRelationshipCheckpoint\")\n .awaitTermination()\n\n assertEquals(\n expected,\n selectRowsFromTable(\"select * from readStreamWithRelationshipCheckpoint order by `rel.timestamp`\", mapLikes)\n )\n }\n\n @Test\n def testReadStreamWithQuery(): Unit = {\n createPersonNodes(0, 1)\n\n val stream = ss.readStream.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"streaming.from\", \"NOW\")\n .option(\"streaming.property.name\", \"timestamp\")\n .option(\n \"query\",\n \"\"\"\n |MATCH (p:Person)\n |WHERE p.timestamp > $stream.offset\n |RETURN p.age AS age, p.timestamp AS timestamp\n |\"\"\".stripMargin\n )\n .option(\n \"streaming.query.offset\",\n \"\"\"\n |MATCH (p:Person)\n |RETURN max(p.timestamp)\n |\"\"\".stripMargin\n )\n .load()\n\n query = stream.writeStream\n .format(\"memory\")\n .queryName(\"readStreamWithQuery\")\n .start()\n\n val total = 60\n val expected: Seq[Map[String, Any]] = (1 to total).map(index =>\n Map(\n \"age\" -> s\"$index\"\n )\n )\n\n Executors.newSingleThreadExecutor().submit(new Runnable {\n override def run(): Unit = {\n createPersonNodes(1, total, 1000, 200)\n }\n })\n\n Assert.assertEventually(\n new Assert.ThrowingSupplier[Seq[Map[String, Any]], Exception] {\n override def get(): Seq[Map[String, Any]] = {\n selectRowsFromTable(\"select * from readStreamWithQuery order by timestamp\", mapPerson)\n }\n },\n Matchers.equalTo(expected),\n 30L,\n TimeUnit.SECONDS\n )\n }\n\n @Test\n def testReadStreamWithQueryGetAll(): Unit = {\n createPersonNodes(0, 1)\n\n val stream = ss.readStream.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"streaming.property.name\", \"timestamp\")\n .option(\"streaming.from\", \"ALL\")\n .option(\n \"query\",\n \"\"\"\n |MATCH (p:Person)\n |WHERE p.timestamp > $stream.offset\n |RETURN p.age AS age, p.timestamp AS timestamp\n |\"\"\".stripMargin\n )\n .option(\n \"streaming.query.offset\",\n \"\"\"\n |MATCH (p:Person)\n |RETURN max(p.timestamp)\n |\"\"\".stripMargin\n )\n .load()\n\n query = stream.writeStream\n .format(\"memory\")\n .queryName(\"readStreamWithQueryAll\")\n .start()\n\n val total = 60\n val expected: Seq[Map[String, Any]] = (0 to total).map(index =>\n Map(\n \"age\" -> s\"$index\"\n )\n ).toList\n\n Executors.newSingleThreadExecutor().submit(new Runnable {\n override def run(): Unit = {\n createPersonNodes(1, total, 1000, 200)\n }\n })\n\n Assert.assertEventually(\n new Assert.ThrowingSupplier[Seq[Map[String, Any]], Exception] {\n override def get(): Seq[Map[String, Any]] = {\n selectRowsFromTable(\"select * from readStreamWithQueryAll order by timestamp\", mapPerson)\n }\n },\n Matchers.equalTo(expected),\n 30L,\n TimeUnit.SECONDS\n )\n }\n\n @Test\n def testReadStreamWithQueryResumesFromCheckpoint(): Unit = {\n createPersonNodes(0, 1)\n\n val stream = ss.readStream.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"streaming.property.name\", \"timestamp\")\n .option(\"streaming.from\", \"ALL\")\n .option(\n \"query\",\n \"\"\"\n |MATCH (p:Person)\n |WHERE p.timestamp > $stream.offset\n |RETURN p.age AS age, p.timestamp AS timestamp\n |\"\"\".stripMargin\n )\n .option(\n \"streaming.query.offset\",\n \"\"\"\n |MATCH (p:Person)\n |RETURN max(p.timestamp)\n |\"\"\".stripMargin\n )\n .load()\n\n val total = 60\n val expected: Seq[Map[String, Any]] = (0 to total).map(index =>\n Map(\n \"age\" -> s\"$index\"\n )\n ).toList\n\n val checkpoint = folder.newFolder()\n\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithQueryCheckpoint\")\n .awaitTermination()\n\n val partial: Int = total / 2\n // create partial number of persons starting from 1\n createPersonNodes(1, partial, 0, 10)\n\n // fetch whatever is available\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithQueryCheckpoint\")\n .awaitTermination()\n\n // create rest of the persons starting from partial+1\n createPersonNodes(partial + 1, total - partial, 0, 10)\n\n // fetch rest of the items from where we left off\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithQueryCheckpoint\")\n .awaitTermination()\n\n assertEquals(\n expected,\n selectRowsFromTable(\"select * from readStreamWithQueryCheckpoint order by timestamp\", mapPerson)\n )\n }\n\n @Test\n def testReadStreamWithQueryResumesFromCheckpointWithNewParams(): Unit = {\n createPersonNodes(0, 1)\n\n val stream = ss.readStream.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"streaming.property.name\", \"timestamp\")\n .option(\"streaming.from\", \"ALL\")\n .option(\n \"query\",\n \"\"\"\n |MATCH (p:Person)\n |WHERE p.timestamp > $stream.from AND p.timestamp <= $stream.to\n |RETURN p.age AS age, p.timestamp AS timestamp\n |\"\"\".stripMargin\n )\n .option(\n \"streaming.query.offset\",\n \"\"\"\n |MATCH (p:Person)\n |RETURN max(p.timestamp)\n |\"\"\".stripMargin\n )\n .load()\n\n val total = 60\n val expected: Seq[Map[String, Any]] = (0 to total).map(index =>\n Map(\n \"age\" -> s\"$index\"\n )\n ).toList\n\n val checkpoint = folder.newFolder()\n\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithQueryCheckpointNewParams\")\n .awaitTermination()\n\n val partial: Int = total / 2\n // create partial number of persons starting from 1\n createPersonNodes(1, partial, 0, 10)\n\n // fetch whatever is available\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithQueryCheckpointNewParams\")\n .awaitTermination()\n\n // create rest of the persons starting from partial+1\n createPersonNodes(partial + 1, total - partial, 0, 10)\n\n // fetch rest of the items from where we left off\n stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"readStreamWithQueryCheckpointNewParams\")\n .awaitTermination()\n\n assertEquals(\n expected,\n selectRowsFromTable(\"select * from readStreamWithQueryCheckpointNewParams order by timestamp\", mapPerson)\n )\n }\n\n @Test\n def testStreamDoesNotReadAnyDataIfStreamingQueryReturnsNothing(): Unit = {\n createPersonNodes(0, 50)\n use(session()) { session =>\n session.run(\"MATCH (p:Person) SET p:Human\").consume()\n }\n val stream = ss.readStream.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"streaming.property.name\", \"timestamp\")\n .option(\"streaming.from\", \"ALL\")\n .option(\n \"query\",\n \"\"\"\n |MATCH (p:Person)\n |WHERE p.timestamp > $stream.from AND p.timestamp <= $stream.to\n |RETURN p.age AS age, p.timestamp AS timestamp\n |\"\"\".stripMargin\n )\n .option(\"streaming.query.offset\", \"MATCH (p:Human) RETURN max(p.timestamp)\")\n .load()\n\n val checkpoint = folder.newFolder()\n // 1st trigger: every node was processed\n var streamTable = stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"testStreamDoesNotReadAnyDataIfStreamingQueryReturnsNothing\")\n streamTable.awaitTermination()\n val firstSource = streamTable.lastProgress.sources.head\n assertEquals(50, firstSource.numInputRows)\n val endOffset1 = firstSource.endOffset\n use(session()) { session =>\n session.run(\"MATCH (p:Human) REMOVE p:Human\").consume()\n }\n // 2nd trigger: previous offset is kept intact\n streamTable = stream.writeStream\n .trigger(Trigger.AvailableNow())\n .option(\"checkpointLocation\", checkpoint.getAbsolutePath)\n .toTable(\"testStreamDoesNotReadAnyDataIfStreamingQueryReturnsNothing\")\n streamTable.awaitTermination()\n val source = streamTable.lastProgress.sources.head\n assertEquals(endOffset1, source.startOffset)\n assertEquals(endOffset1, source.endOffset)\n assertEquals(0L, source.numInputRows)\n }\n\n private def createPersonNodes(minAge: Int, maxAge: Int, delayMs: Int = 0, intervalMs: Int = 0): Unit = {\n use(session()) { session =>\n Thread.sleep(delayMs)\n (minAge until minAge + maxAge).foreach(age => {\n Thread.sleep(intervalMs)\n session.run(\n s\"CREATE (p:Person {age: '$age', timestamp: timestamp()})\"\n ).consume()\n })\n }\n }\n\n private def createMovieNodes(from: Int, count: Int, delayMs: Int = 0, intervalMs: Int = 0): Unit = {\n use(session()) { session =>\n Thread.sleep(delayMs)\n (from until from + count).foreach(index => {\n Thread.sleep(intervalMs)\n session.run(\n s\"CREATE (n:Movie {title: 'My movie $index', timestamp: timestamp()})\"\n ).consume()\n })\n }\n }\n\n private def createLikesRelationships(from: Int, count: Int, delayMs: Int = 0, intervalMs: Int = 0): Unit = {\n use(session()) { session =>\n Thread.sleep(delayMs)\n (from until from + count).foreach(index => {\n Thread.sleep(intervalMs)\n session.run(\n s\"\"\"\n |CREATE (person:Person {age: $index})\n |CREATE (post:Post {hash: \"hash$index\"})\n |CREATE (person)-[:LIKES{id: $index, timestamp: timestamp()}]->(post)\n |\"\"\".stripMargin\n ).consume()\n })\n }\n }\n\n private def selectRowsFromTable(\n query: String,\n mapper: (Row) => Map[String, Any]\n ): Seq[Map[String, Any]] = {\n ss.sql(query)\n .collect()\n .map(row => mapper(row))\n .toList\n }\n\n private def mapPerson(row: Row): Map[String, Any] = {\n Map(\n \"age\" -> row.getAs[String](\"age\")\n )\n }\n\n private def mapMovie(row: Row): Map[String, Any] = {\n Map(\n \"\" -> row.getAs[java.util.List[String]](\"\"),\n \"title\" -> row.getAs[String](\"title\")\n )\n }\n\n private def mapLikes(row: Row): Map[String, Any] = {\n Map(\n \"\" -> row.getAs[String](\"\"),\n \"\" -> row.getAs[java.util.List[String]](\"\"),\n \"source.age\" -> row.getAs[Long](\"source.age\"),\n \"\" -> row.getAs[java.util.List[String]](\"\"),\n \"target.hash\" -> row.getAs[String](\"target.hash\"),\n \"rel.id\" -> row.getAs[Long](\"rel.id\")\n )\n }\n\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DataSourceStreamingWriterTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.sql.execution.streaming.MemoryStream\nimport org.apache.spark.sql.streaming.StreamingQuery\nimport org.hamcrest.Matchers\nimport org.junit.After\nimport org.junit.Test\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.spark.Assert.ThrowingSupplier\n\nimport java.util.UUID\nimport java.util.concurrent.TimeUnit\n\nclass DataSourceStreamingWriterTSE extends SparkConnectorScalaBaseTSE {\n\n private var query: StreamingQuery = null\n\n @After\n def close(): Unit = {\n if (query != null) {\n query.stop()\n }\n }\n\n @Test\n def testSinkStreamWithLabelsWithAppend(): Unit = {\n implicit val ctx = ss.sqlContext\n import ss.implicits._\n val memStream = MemoryStream[Int]\n val recordSize = 2000\n val partition = 5\n val checkpointLocation = \"/tmp/checkpoint/\" + UUID.randomUUID().toString\n query = memStream.toDF().writeStream\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"save.mode\", \"Append\")\n .option(\"labels\", \"Timestamp\")\n .option(\"checkpointLocation\", checkpointLocation)\n .option(\"node.keys\", \"value\")\n .start()\n\n (1 to partition).foreach(index => {\n // we send the total of records in 5 times\n val start = ((index - 1) * recordSize) + 1\n val end = index * recordSize\n memStream.addData((start to end).toArray)\n })\n\n Assert.assertEventually(\n new ThrowingSupplier[Boolean, Exception] {\n override def get(): Boolean = {\n val dataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Timestamp\")\n .load()\n\n val collect = dataFrame.collect()\n val data = if (dataFrame.columns.contains(\"value\")) {\n collect\n .map(row => row.getAs[Long](\"value\").toInt)\n .sorted\n } else {\n Array.empty[Int]\n }\n data.toList == (1 to (recordSize * partition)).toList\n }\n },\n Matchers.equalTo(true),\n 30L,\n TimeUnit.SECONDS\n )\n }\n\n @Test\n def testSinkStreamWithRelationshipWithAppend(): Unit = {\n implicit val ctx = ss.sqlContext\n import ss.implicits._\n val memStream = MemoryStream[Int]\n val recordSize = 2000\n val partition = 5\n val checkpointLocation = \"/tmp/checkpoint/\" + UUID.randomUUID().toString\n\n query = memStream.toDF().writeStream\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"save.mode\", \"Append\")\n .option(\"relationship\", \"PAIRS\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":From\")\n .option(\"relationship.source.node.keys\", \"value\")\n .option(\"relationship.source.save.mode\", \"Append\")\n .option(\"relationship.target.labels\", \":To\")\n .option(\"relationship.target.node.keys\", \"value\")\n .option(\"relationship.target.save.mode\", \"Append\")\n .option(\"checkpointLocation\", checkpointLocation)\n .start()\n\n (1 to partition).foreach(index => {\n // we send the total of records in 5 times\n val start = ((index - 1) * recordSize) + 1\n val end = index * recordSize\n memStream.addData((start to end).toArray)\n })\n\n Assert.assertEventually(\n new ThrowingSupplier[Boolean, Exception] {\n override def get(): Boolean =\n try {\n val dataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"PAIRS\")\n .option(\"relationship.source.labels\", \":From\")\n .option(\"relationship.target.labels\", \":To\")\n .load()\n\n val collect = dataFrame.collect()\n val data = if (dataFrame.columns.contains(\"source.value\") && dataFrame.columns.contains(\"target.value\")) {\n collect\n .map(row => (row.getAs[Long](\"source.value\").toInt, row.getAs[Long](\"target.value\").toInt))\n .sorted\n } else {\n Array.empty[(Int, Int)]\n }\n data.toList == (1 to (recordSize * partition)).map(v => (v, v)).toList\n } catch {\n case _: Throwable => false\n }\n },\n Matchers.equalTo(true),\n 30L,\n TimeUnit.SECONDS\n )\n }\n\n @Test\n def testSinkStreamWithQuery(): Unit = {\n implicit val ctx = ss.sqlContext\n import ss.implicits._\n val memStream = MemoryStream[Int]\n val recordSize = 2000\n val partition = 5\n val checkpointLocation = \"/tmp/checkpoint/\" + UUID.randomUUID().toString\n\n query = memStream.toDF().writeStream\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MERGE (m:MyNewNode {the_value: event.value})\")\n .option(\"checkpointLocation\", checkpointLocation)\n .start()\n\n (1 to partition).foreach(index => {\n // we send the total of records in 5 times\n val start = ((index - 1) * recordSize) + 1\n val end = index * recordSize\n memStream.addData((start to end).toArray)\n })\n\n Assert.assertEventually(\n new ThrowingSupplier[Boolean, Exception] {\n override def get(): Boolean =\n try {\n val dataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"MyNewNode\")\n .load()\n\n val collect = dataFrame.collect()\n val data = if (dataFrame.columns.contains(\"the_value\")) {\n collect\n .map(row => row.getAs[Long](\"the_value\").toInt)\n .sorted\n } else {\n Array.empty[Int]\n }\n val l1 = data.toList\n val l2 = (1 to (recordSize * partition)).map(v => v).toList\n l1 == l2\n } catch {\n case _: Throwable => false\n }\n },\n Matchers.equalTo(true),\n 30L,\n TimeUnit.SECONDS\n )\n }\n\n @Test\n def testSinkStreamWithLabelsWithOverwrite(): Unit = {\n implicit val ctx = ss.sqlContext\n import ss.implicits._\n val memStream = MemoryStream[Int]\n val partition = 5\n val checkpointLocation = \"/tmp/checkpoint/\" + UUID.randomUUID().toString\n\n SparkConnectorScalaSuiteIT.session().run(\n \"CREATE CONSTRAINT timestamp_value FOR (t:Timestamp) REQUIRE (t.value) IS UNIQUE\"\n )\n\n query = memStream.toDF().writeStream\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"save.mode\", \"Overwrite\")\n .option(\"labels\", \"Timestamp\")\n .option(\"checkpointLocation\", checkpointLocation)\n .option(\"node.keys\", \"value\")\n .start()\n (1 to partition).foreach(index => {\n memStream.addData((1 to 500).toArray)\n })\n\n Assert.assertEventually(\n new ThrowingSupplier[Boolean, Exception] {\n override def get(): Boolean = {\n val dataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Timestamp\")\n .load()\n\n val collect = dataFrame.collect()\n val data = if (dataFrame.columns.contains(\"value\")) {\n collect\n .map(row => row.getAs[Long](\"value\").toInt)\n .sorted\n } else {\n Array.empty[Int]\n }\n data.toList == (1 to 500).toList\n }\n },\n Matchers.equalTo(true),\n 30L,\n TimeUnit.SECONDS\n )\n\n SparkConnectorScalaSuiteIT.session().run(\"DROP CONSTRAINT timestamp_value\")\n }\n\n @Test\n def testSinkStreamWithRelationshipWithAppendAndOverwrite(): Unit = {\n implicit val ctx = ss.sqlContext\n import ss.implicits._\n val memStream = MemoryStream[Int]\n val partition = 5\n val checkpointLocation = \"/tmp/checkpoint/\" + UUID.randomUUID().toString\n\n SparkConnectorScalaSuiteIT.driver.session()\n .writeTransaction(\n new TransactionWork[Unit] {\n override def execute(tx: Transaction): Unit = {\n tx.run(\"CREATE CONSTRAINT From_value FOR (p:From) REQUIRE p.value IS UNIQUE\")\n tx.run(\"CREATE CONSTRAINT To_value FOR (p:To) REQUIRE p.value IS UNIQUE\")\n }\n }\n )\n\n query = memStream.toDF().writeStream\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"save.mode\", \"Append\")\n .option(\"relationship\", \"PAIRS\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":From\")\n .option(\"relationship.source.node.keys\", \"value\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.target.labels\", \":To\")\n .option(\"relationship.target.node.keys\", \"value\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"checkpointLocation\", checkpointLocation)\n .start()\n\n (1 to partition).foreach(index => {\n memStream.addData((1 to 500).toArray)\n })\n\n Assert.assertEventually(\n new ThrowingSupplier[Boolean, Exception] {\n override def get(): Boolean =\n try {\n val dataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"PAIRS\")\n .option(\"relationship.source.labels\", \":From\")\n .option(\"relationship.target.labels\", \":To\")\n .load()\n\n val collect = dataFrame.collect()\n val data = if (dataFrame.columns.contains(\"source.value\") && dataFrame.columns.contains(\"target.value\")) {\n collect\n .map(row => (row.getAs[Long](\"source.value\").toInt, row.getAs[Long](\"target.value\").toInt))\n .sorted\n } else {\n Array.empty[(Int, Int)]\n }\n data.toList == (1 to 500).flatMap(v => (1 to 5).map(_ => (v, v)))\n } catch {\n case _: Throwable => false\n }\n },\n Matchers.equalTo(true),\n 30L,\n TimeUnit.SECONDS\n )\n\n SparkConnectorScalaSuiteIT.driver.session()\n .writeTransaction(\n new TransactionWork[Unit] {\n override def execute(tx: Transaction): Unit = {\n tx.run(\"DROP CONSTRAINT From_value\")\n tx.run(\"DROP CONSTRAINT To_value\")\n }\n }\n )\n }\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DataSourceWriterNeo4jSkipNullKeysTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.SparkException\nimport org.apache.spark.sql.SaveMode\nimport org.junit.Assume\nimport org.junit.Test\nimport org.neo4j.Closeables.use\nimport org.neo4j.caniuse.CanIUse\nimport org.neo4j.caniuse.Schema\n\nclass DataSourceWriterNeo4jSkipNullKeysTSE extends SparkConnectorScalaBaseTSE {\n\n import ss.implicits._\n\n @Test\n def `fails to write nodes when key properties contain null values`(): Unit = {\n val cities = Seq(\n (Some(1), \"Cherbourg en Cotentin\"),\n (Some(2), \"London\"),\n (Some(3), \"Malmö\"),\n (None, \"Moon\")\n ).toDF(\"id\", \"city\")\n\n val caught = intercept[SparkException] {\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":City\")\n .option(\"node.keys\", \"id\")\n .option(\"schema.optimization.node.keys\", \"KEY\")\n .save()\n }\n assert(caught.getMessage contains \"Cannot merge the following node because of null property value\")\n }\n\n @Test\n def `fails to write relationships when source node key properties contain null values`(): Unit = {\n val caught = intercept[SparkException] {\n val cities = Seq(\n (Some(1), Some(2), \"British Airways\"),\n (Some(2), Some(3), \"Turkish Airlines\"),\n (None, Some(5), \"Another Airline\")\n ).toDF(\"from\", \"to\", \"airline\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"FLIES_TO\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.source.labels\", \":City\")\n .option(\"relationship.source.node.keys\", \"from:id\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.target.labels\", \":City\")\n .option(\"relationship.target.node.keys\", \"to:id\")\n .option(\"relationship.properties\", \"airline\")\n .option(\"schema.optimization.node.keys\", \"KEY\")\n .save()\n }\n assert(caught.getMessage contains \"Cannot merge the following node because of null property value\")\n }\n\n @Test\n def `fails to write relationships when target node key properties contain null values`(): Unit = {\n val caught = intercept[SparkException] {\n val cities = Seq(\n (Some(1), Some(2), \"British Airways\"),\n (Some(2), Some(3), \"Turkish Airlines\"),\n (Some(3), None, \"Another Airline\")\n ).toDF(\"from\", \"to\", \"airline\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"FLIES_TO\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.source.labels\", \":City\")\n .option(\"relationship.source.node.keys\", \"from:id\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.target.labels\", \":City\")\n .option(\"relationship.target.node.keys\", \"to:id\")\n .option(\"relationship.properties\", \"airline\")\n .option(\"schema.optimization.node.keys\", \"KEY\")\n .save()\n }\n assert(caught.getMessage contains \"Cannot merge the following node because of null property value\")\n }\n\n @Test\n def `fails to write relationships when relationship key properties contain null values`(): Unit = {\n Assume.assumeTrue(\n CanIUse.INSTANCE.canIUse(Schema.INSTANCE.relationshipKeyConstraints()).withNeo4j(SparkConnectorScalaSuiteIT.neo4j)\n )\n\n val caught = intercept[SparkException] {\n val cities = Seq(\n (Some(1), Some(2), Some(\"BA721\"), \"British Airways\"),\n (Some(2), Some(3), Some(\"TK211\"), \"Turkish Airlines\"),\n (Some(3), Some(4), None, \"Another Airline\")\n ).toDF(\"from\", \"to\", \"flight\", \"airline\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"FLIES_TO\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.source.labels\", \":City\")\n .option(\"relationship.source.node.keys\", \"from:id\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.target.labels\", \":City\")\n .option(\"relationship.target.node.keys\", \"to:id\")\n .option(\"relationship.keys\", \"flight\")\n .option(\"relationship.properties\", \"airline\")\n .option(\"schema.optimization.node.keys\", \"KEY\")\n .option(\"schema.optimization.relationship.keys\", \"KEY\")\n .save()\n }\n assert(caught.getMessage contains \"Cannot merge the following relationship because of null property value\")\n }\n\n @Test\n def `skips nodes when key properties contain null values with APPEND mode`(): Unit = {\n val cities = Seq(\n (Some(1), \"Cherbourg en Cotentin\"),\n (Some(2), \"London\"),\n (Some(3), \"Malmö\"),\n (None, \"Moon\")\n ).toDF(\"id\", \"city\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":City\")\n .option(\"node.keys\", \"id\")\n .option(\"node.keys.skip.nulls\", \"true\")\n .save()\n\n use(SparkConnectorScalaSuiteIT.driver.session()) { session =>\n val result = session.run(\"MATCH (n:City) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(result == 3)\n }\n }\n\n @Test\n def `skips nodes when key properties contain null values with OVERWRITE mode`(): Unit = {\n val cities = Seq(\n (Some(1), \"Cherbourg en Cotentin\"),\n (Some(2), \"London\"),\n (Some(3), \"Malmö\"),\n (None, \"Moon\")\n ).toDF(\"id\", \"city\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":City\")\n .option(\"node.keys\", \"id\")\n .option(\"node.keys.skip.nulls\", \"true\")\n .option(\"schema.optimization.node.keys\", \"KEY\")\n .save()\n\n use(SparkConnectorScalaSuiteIT.driver.session()) { session =>\n val result = session.run(\"MATCH (n:City) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(result == 3)\n }\n }\n\n @Test\n def `skips relationships when source or target node key properties contain null values`(): Unit = {\n val cities = Seq(\n (Some(1), Some(2), \"British Airways\"),\n (Some(2), Some(3), \"Turkish Airlines\"),\n (None, Some(5), \"Another Airline\"),\n (Some(5), None, \"Another Airline\")\n ).toDF(\"from\", \"to\", \"airline\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"FLIES_TO\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.source.labels\", \":City\")\n .option(\"relationship.source.node.keys\", \"from:id\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.target.labels\", \":City\")\n .option(\"relationship.target.node.keys\", \"to:id\")\n .option(\"relationship.properties\", \"airline\")\n .option(\"schema.optimization.node.keys\", \"KEY\")\n .option(\"relationship.source.node.keys.skip.nulls\", \"true\")\n .option(\"relationship.target.node.keys.skip.nulls\", \"true\")\n .save()\n\n use(SparkConnectorScalaSuiteIT.driver.session()) { session =>\n val cities = session.run(\"MATCH (n:City) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(cities == 3)\n\n val citiesWithId5 = session.run(\"MATCH (n:City {id: 5}) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(citiesWithId5 == 0)\n\n val flies = session.run(\"MATCH ()-[r:FLIES_TO]->() RETURN count(r) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(flies == 2)\n }\n }\n\n @Test\n def `skips relationships when source or target node key properties contain null values when nodes are matched`()\n : Unit = {\n use(SparkConnectorScalaSuiteIT.driver.session()) { session =>\n session.run(\"UNWIND [1,2,3,5] AS id CREATE (:City {id: id})\").consume()\n }\n\n val cities = Seq(\n (Some(1), Some(2), \"British Airways\"),\n (Some(2), Some(3), \"Turkish Airlines\"),\n (None, Some(5), \"Another Airline\"),\n (Some(5), None, \"Another Airline\")\n ).toDF(\"from\", \"to\", \"airline\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"FLIES_TO\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Match\")\n .option(\"relationship.source.labels\", \":City\")\n .option(\"relationship.source.node.keys\", \"from:id\")\n .option(\"relationship.target.save.mode\", \"Match\")\n .option(\"relationship.target.labels\", \":City\")\n .option(\"relationship.target.node.keys\", \"to:id\")\n .option(\"relationship.properties\", \"airline\")\n .option(\"relationship.source.node.keys.skip.nulls\", \"true\")\n .option(\"relationship.target.node.keys.skip.nulls\", \"true\")\n .save()\n\n use(SparkConnectorScalaSuiteIT.driver.session()) { session =>\n val cities = session.run(\"MATCH (n:City) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(cities == 4)\n\n val flies = session.run(\"MATCH ()-[r:FLIES_TO]->() RETURN count(r) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(flies == 2)\n }\n }\n\n @Test\n def `skips relationships when source or target node key properties contain null values when nodes are appended`()\n : Unit = {\n val cities = Seq(\n (Some(1), Some(2), \"British Airways\"),\n (Some(3), Some(4), \"Turkish Airlines\"),\n (None, Some(5), \"Another Airline\"),\n (Some(5), None, \"Another Airline\")\n ).toDF(\"from\", \"to\", \"airline\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"FLIES_TO\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Append\")\n .option(\"relationship.source.labels\", \":City\")\n .option(\"relationship.source.node.keys\", \"from:id\")\n .option(\"relationship.target.save.mode\", \"Append\")\n .option(\"relationship.target.labels\", \":City\")\n .option(\"relationship.target.node.keys\", \"to:id\")\n .option(\"relationship.properties\", \"airline\")\n .option(\"relationship.source.node.keys.skip.nulls\", \"true\")\n .option(\"relationship.target.node.keys.skip.nulls\", \"true\")\n .save()\n\n use(SparkConnectorScalaSuiteIT.driver.session()) { session =>\n val cities = session.run(\"MATCH (n:City) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(cities == 4)\n\n val citiesWithId5 = session.run(\"MATCH (n:City {id: 5}) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(citiesWithId5 == 0)\n\n val flies = session.run(\"MATCH ()-[r:FLIES_TO]->() RETURN count(r) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(flies == 2)\n }\n }\n\n @Test\n def `skips relationships when source or target node key properties contain null values with append mode`(): Unit = {\n val cities = Seq(\n (Some(1), Some(2), \"British Airways\"),\n (Some(3), Some(4), \"Turkish Airlines\"),\n (None, Some(5), \"Another Airline\"),\n (Some(5), None, \"Another Airline\")\n ).toDF(\"from\", \"to\", \"airline\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"FLIES_TO\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.source.labels\", \":City\")\n .option(\"relationship.source.node.keys\", \"from:id\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.target.labels\", \":City\")\n .option(\"relationship.target.node.keys\", \"to:id\")\n .option(\"relationship.properties\", \"airline\")\n .option(\"schema.optimization.node.keys\", \"KEY\")\n .option(\"relationship.source.node.keys.skip.nulls\", \"true\")\n .option(\"relationship.target.node.keys.skip.nulls\", \"true\")\n .save()\n\n use(SparkConnectorScalaSuiteIT.driver.session()) { session =>\n val cities = session.run(\"MATCH (n:City) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(cities == 4)\n\n val citiesWithId5 = session.run(\"MATCH (n:City {id: 5}) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(citiesWithId5 == 0)\n\n val flies = session.run(\"MATCH ()-[r:FLIES_TO]->() RETURN count(r) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(flies == 2)\n }\n }\n\n @Test\n def `skips relationships when source or target node key properties contain null values when nodes are matched with append mode`()\n : Unit = {\n use(SparkConnectorScalaSuiteIT.driver.session()) { session =>\n session.run(\"UNWIND [1,2,3,5] AS id CREATE (:City {id: id})\").consume()\n }\n\n val cities = Seq(\n (Some(1), Some(2), \"British Airways\"),\n (Some(2), Some(3), \"Turkish Airlines\"),\n (None, Some(5), \"Another Airline\"),\n (Some(5), None, \"Another Airline\")\n ).toDF(\"from\", \"to\", \"airline\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"FLIES_TO\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Match\")\n .option(\"relationship.source.labels\", \":City\")\n .option(\"relationship.source.node.keys\", \"from:id\")\n .option(\"relationship.target.save.mode\", \"Match\")\n .option(\"relationship.target.labels\", \":City\")\n .option(\"relationship.target.node.keys\", \"to:id\")\n .option(\"relationship.properties\", \"airline\")\n .option(\"relationship.source.node.keys.skip.nulls\", \"true\")\n .option(\"relationship.target.node.keys.skip.nulls\", \"true\")\n .save()\n\n use(SparkConnectorScalaSuiteIT.driver.session()) { session =>\n val cities = session.run(\"MATCH (n:City) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(cities == 4)\n\n val flies = session.run(\"MATCH ()-[r:FLIES_TO]->() RETURN count(r) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(flies == 2)\n }\n }\n\n @Test\n def `skips relationships when source or target node key properties contain null values when nodes are appended with append mode`()\n : Unit = {\n val cities = Seq(\n (Some(1), Some(2), \"British Airways\"),\n (Some(3), Some(4), \"Turkish Airlines\"),\n (None, Some(5), \"Another Airline\"),\n (Some(5), None, \"Another Airline\")\n ).toDF(\"from\", \"to\", \"airline\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"FLIES_TO\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Append\")\n .option(\"relationship.source.labels\", \":City\")\n .option(\"relationship.source.node.keys\", \"from:id\")\n .option(\"relationship.target.save.mode\", \"Append\")\n .option(\"relationship.target.labels\", \":City\")\n .option(\"relationship.target.node.keys\", \"to:id\")\n .option(\"relationship.properties\", \"airline\")\n .option(\"relationship.source.node.keys.skip.nulls\", \"true\")\n .option(\"relationship.target.node.keys.skip.nulls\", \"true\")\n .save()\n\n use(SparkConnectorScalaSuiteIT.driver.session()) { session =>\n val cities = session.run(\"MATCH (n:City) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(cities == 4)\n\n val citiesWithId5 = session.run(\"MATCH (n:City {id: 5}) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(citiesWithId5 == 0)\n\n val flies = session.run(\"MATCH ()-[r:FLIES_TO]->() RETURN count(r) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(flies == 2)\n }\n }\n\n @Test\n def `skips relationships when relationship key properties contain null values`(): Unit = {\n Assume.assumeTrue(\n CanIUse.INSTANCE.canIUse(Schema.INSTANCE.relationshipKeyConstraints()).withNeo4j(SparkConnectorScalaSuiteIT.neo4j)\n )\n\n val cities = Seq(\n (Some(1), Some(2), Some(\"BA721\"), \"British Airways\"),\n (Some(2), Some(3), Some(\"TK211\"), \"Turkish Airlines\"),\n (Some(3), Some(5), None, \"Another Airline\")\n ).toDF(\"from\", \"to\", \"flight\", \"airline\")\n\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"FLIES_TO\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.source.labels\", \":City\")\n .option(\"relationship.source.node.keys\", \"from:id\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.target.labels\", \":City\")\n .option(\"relationship.target.node.keys\", \"to:id\")\n .option(\"relationship.keys\", \"flight\")\n .option(\"relationship.properties\", \"airline\")\n .option(\"schema.optimization.node.keys\", \"KEY\")\n .option(\"schema.optimization.relationship.keys\", \"KEY\")\n .option(\"relationship.keys.skip.nulls\", \"true\")\n .save()\n\n use(SparkConnectorScalaSuiteIT.driver.session()) { session =>\n val cities = session.run(\"MATCH (n:City) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(cities == 3)\n\n val citiesWithId5 = session.run(\"MATCH (n:City {id: 5}) RETURN count(n) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(citiesWithId5 == 0)\n\n val flies = session.run(\"MATCH ()-[r:FLIES_TO]->() RETURN count(r) as count\")\n .single()\n .get(\"count\")\n .asLong()\n assert(flies == 2)\n }\n }\n\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DataSourceWriterNeo4jTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.commons.lang3.exception.ExceptionUtils\nimport org.apache.spark.scheduler.SparkListener\nimport org.apache.spark.scheduler.SparkListenerStageCompleted\nimport org.apache.spark.scheduler.SparkListenerStageSubmitted\nimport org.apache.spark.sql.DataFrame\nimport org.apache.spark.sql.SaveMode\nimport org.hamcrest.Matchers\nimport org.junit.Assert.assertEquals\nimport org.junit.Assert.assertNotNull\nimport org.junit.Assert.assertTrue\nimport org.junit.Test\nimport org.neo4j.Closeables.use\nimport org.neo4j.driver.Session\nimport org.neo4j.driver.SessionConfig\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.summary.ResultSummary\nimport org.neo4j.spark.writer.DataWriterMetrics\n\nimport java.util.concurrent.TimeUnit\nimport java.util.concurrent.atomic.AtomicReference\n\nclass DataSourceWriterNeo4jTSE extends SparkConnectorScalaBaseTSE {\n\n import ss.implicits._\n\n @Test\n def `should read and write relations with append mode`(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * $total, name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n use(SparkConnectorScalaSuiteIT.session(\"system\")) { session =>\n session.run(\"CREATE OR REPLACE DATABASE db1 WAIT 30 seconds\").consume()\n session.run(\"CREATE OR REPLACE DATABASE db2 WAIT 30 seconds\").consume()\n }\n\n use(SparkConnectorScalaSuiteIT.session(\"db1\")) { session =>\n session.run(fixtureQuery).consume()\n }\n\n use(SparkConnectorScalaSuiteIT.session(\"db2\")) { session =>\n session\n .writeTransaction(\n new TransactionWork[Unit] {\n override def execute(tx: Transaction): Unit = {\n tx.run(\"CREATE CONSTRAINT person_id FOR (p:Person) REQUIRE p.id IS UNIQUE\")\n tx.run(\"CREATE CONSTRAINT product_id FOR (p:Product) REQUIRE p.id IS UNIQUE\")\n }\n }\n )\n }\n\n try {\n val dfOriginal: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n\n dfOriginal.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"relationship\", \"SOLD\")\n .option(\"relationship.save.strategy\", \"NATIVE\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.source.save.mode\", \"Append\")\n .option(\"relationship.target.labels\", \":Product\")\n .option(\"relationship.target.save.mode\", \"Append\")\n .option(\"batch.size\", \"11\")\n .save()\n\n // let's write again to prove that 2 relationship are being added\n dfOriginal.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"relationship\", \"SOLD\")\n .option(\"relationship.save.strategy\", \"NATIVE\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.source.node.keys\", \"source.id:id\")\n .option(\"relationship.target.labels\", \":Product\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.target.node.keys\", \"target.id:id\")\n .option(\"batch.size\", \"11\")\n .save()\n\n val dfCopy = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"relationship\", \"SOLD\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n\n val dfOriginalCount = dfOriginal.count()\n assertEquals(dfOriginalCount * 2, dfCopy.count())\n\n val resSourceOrig = dfOriginal.select(\"`source.id`\").orderBy(\"`source.id`\").collectAsList()\n val resSourceCopy = dfCopy.select(\"`source.id`\").orderBy(\"`source.id`\").collectAsList()\n val resTargetOrig = dfOriginal.select(\"`target.id`\").orderBy(\"`target.id`\").collectAsList()\n val resTargetCopy = dfCopy.select(\"`target.id`\").orderBy(\"`target.id`\").collectAsList()\n\n for (i <- 0 until 1) {\n assertEquals(\n resSourceOrig.get(i).getLong(0),\n resSourceCopy.get(i).getLong(0)\n )\n assertEquals(\n resTargetOrig.get(i).getLong(0),\n resTargetCopy.get(i).getLong(0)\n )\n }\n\n assertEquals(\n 2,\n dfCopy.where(\"`source.id` = 1\").count()\n )\n } finally {\n SparkConnectorScalaSuiteIT.driver.session(SessionConfig.forDatabase(\"db2\"))\n .writeTransaction(\n new TransactionWork[Unit] {\n override def execute(tx: Transaction): Unit = {\n tx.run(\"DROP CONSTRAINT person_id\")\n tx.run(\"DROP CONSTRAINT product_id\")\n }\n }\n )\n }\n }\n\n @Test\n def `should read and write relations with overwrite mode`(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * $total, name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n use(SparkConnectorScalaSuiteIT.session(\"system\")) { session =>\n session.run(\"CREATE OR REPLACE DATABASE db1 WAIT 30 seconds\").consume()\n session.run(\"CREATE OR REPLACE DATABASE db2 WAIT 30 seconds\").consume()\n }\n\n use(SparkConnectorScalaSuiteIT.session(\"db1\")) { session =>\n session.run(fixtureQuery).consume()\n }\n\n use(SparkConnectorScalaSuiteIT.session(\"db2\")) { session =>\n session\n .writeTransaction(\n new TransactionWork[Unit] {\n override def execute(tx: Transaction): Unit = {\n tx.run(\"CREATE CONSTRAINT person_id FOR (p:Person) REQUIRE p.id IS UNIQUE\")\n tx.run(\"CREATE CONSTRAINT product_id FOR (p:Product) REQUIRE p.id IS UNIQUE\")\n }\n }\n )\n }\n\n try {\n val dfOriginal: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n .orderBy(\"`source.id`\", \"`target.id`\")\n\n dfOriginal.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"relationship\", \"SOLD\")\n .option(\"relationship.save.strategy\", \"NATIVE\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.target.labels\", \":Product\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"batch.size\", \"11\")\n .save()\n\n // let's write the same thing again to prove there will be just one relation\n dfOriginal.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"relationship\", \"SOLD\")\n .option(\"relationship.save.strategy\", \"NATIVE\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.source.node.keys\", \"source.id:id\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.target.labels\", \":Product\")\n .option(\"relationship.target.node.keys\", \"target.id:id\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"batch.size\", \"11\")\n .save()\n\n val dfCopy = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"relationship\", \"SOLD\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n .orderBy(\"`source.id`\", \"`target.id`\")\n\n val dfOriginalCount = dfOriginal.count()\n assertEquals(dfOriginalCount, dfCopy.count())\n\n for (i <- 0 until 1) {\n assertEquals(\n dfOriginal.select(\"`source.id`\").collectAsList().get(i).getLong(0),\n dfCopy.select(\"`source.id`\").collectAsList().get(i).getLong(0)\n )\n assertEquals(\n dfOriginal.select(\"`target.id`\").collectAsList().get(i).getLong(0),\n dfCopy.select(\"`target.id`\").collectAsList().get(i).getLong(0)\n )\n }\n\n assertEquals(\n 1,\n dfCopy.where(\"`source.id` = 1\").count()\n )\n } finally {\n SparkConnectorScalaSuiteIT.driver.session(SessionConfig.forDatabase(\"db2\"))\n .writeTransaction(\n new TransactionWork[Unit] {\n override def execute(tx: Transaction): Unit = {\n tx.run(\"DROP CONSTRAINT person_id\")\n tx.run(\"DROP CONSTRAINT product_id\")\n }\n }\n )\n }\n }\n\n @Test\n def `should read and write relations with MATCH and node keys`(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * $total, name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n use(SparkConnectorScalaSuiteIT.session(\"system\")) { session =>\n session.run(\"CREATE OR REPLACE DATABASE db1 WAIT 30 seconds\").consume()\n session.run(\"CREATE OR REPLACE DATABASE db2 WAIT 30 seconds\").consume()\n }\n\n use(SparkConnectorScalaSuiteIT.session(\"db1\")) { session =>\n session.run(fixtureQuery).consume()\n }\n\n use(SparkConnectorScalaSuiteIT.session(\"db2\")) { session =>\n session.run(fixtureQuery).consume()\n }\n\n val dfOriginal: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n .orderBy(\"`source.id`\", \"`target.id`\")\n\n dfOriginal.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"relationship\", \"SOLD\")\n .option(\"relationship.save.strategy\", \"NATIVE\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .option(\"relationship.source.node.keys\", \"source.id:id\")\n .option(\"relationship.target.node.keys\", \"target.id:id\")\n .option(\"relationship.source.save.mode\", \"Match\")\n .option(\"relationship.target.save.mode\", \"Match\")\n .option(\"batch.size\", \"11\")\n .save()\n\n val dfCopy = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"relationship\", \"SOLD\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n .orderBy(\"`source.id`\", \"`target.id`\")\n\n for (i <- 0 until 1) {\n assertEquals(\n dfOriginal.select(\"`source.id`\").collectAsList().get(i).getLong(0),\n dfCopy.select(\"`source.id`\").collectAsList().get(i).getLong(0)\n )\n assertEquals(\n dfOriginal.select(\"`target.id`\").collectAsList().get(i).getLong(0),\n dfCopy.select(\"`target.id`\").collectAsList().get(i).getLong(0)\n )\n }\n }\n\n @Test\n def `should read and write relations with MERGE and node keys`(): Unit = {\n val total = 100\n val fixtureQuery: String =\n s\"\"\"UNWIND range(1, $total) as id\n |CREATE (pr:Product {id: id * $total, name: 'Product ' + id})\n |CREATE (pe:Person {id: id, fullName: 'Person ' + id})\n |CREATE (pe)-[:BOUGHT{when: rand(), quantity: rand() * 1000}]->(pr)\n |RETURN *\n \"\"\".stripMargin\n\n use(SparkConnectorScalaSuiteIT.session(\"system\")) { session =>\n session.run(\"CREATE OR REPLACE DATABASE db1 WAIT 30 seconds\").consume()\n session.run(\"CREATE OR REPLACE DATABASE db2 WAIT 30 seconds\").consume()\n }\n\n use(SparkConnectorScalaSuiteIT.session(\"db1\")) { session =>\n session.run(fixtureQuery).consume()\n }\n\n val dfOriginal: DataFrame = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"relationship\", \"BOUGHT\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n .orderBy(\"`source.id`\", \"`target.id`\")\n\n dfOriginal.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"relationship\", \"SOLD\")\n .option(\"relationship.save.strategy\", \"NATIVE\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .option(\"relationship.source.node.keys\", \"source.id:id\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.target.node.keys\", \"target.id:id\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"batch.size\", \"11\")\n .save()\n\n val dfCopy = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db2\")\n .option(\"relationship\", \"SOLD\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.target.labels\", \":Product\")\n .load()\n .orderBy(\"`source.id`\", \"`target.id`\")\n\n for (i <- 0 until 1) {\n assertEquals(\n dfOriginal.select(\"`source.id`\").collectAsList().get(i).getLong(0),\n dfCopy.select(\"`source.id`\").collectAsList().get(i).getLong(0)\n )\n assertEquals(\n dfOriginal.select(\"`target.id`\").collectAsList().get(i).getLong(0),\n dfCopy.select(\"`target.id`\").collectAsList().get(i).getLong(0)\n )\n }\n }\n\n @Test\n def `should read relations and write relation with match mode`(): Unit = {\n val fixtureQuery: String =\n s\"\"\"CREATE (m:Musician {name: \"John Bonham\", age: 32})\n |CREATE (i:Instrument {name: \"Drums\"})\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.driver.session(SessionConfig.forDatabase(\"db1\"))\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val musicDf = Seq(\n (12, 32, \"John Bonham\", \"Drums\")\n ).toDF(\"experience\", \"age\", \"name\", \"instrument\")\n\n musicDf.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.save.mode\", \"Match\")\n .option(\"relationship.target.save.mode\", \"Match\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"name,age\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .save()\n\n val df2 = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .load()\n\n val experience = df2.select(\"`source.age`\").where(\"`source.name` = 'John Bonham'\")\n .collectAsList().get(0).getLong(0)\n\n assertEquals(32, experience)\n }\n\n @Test\n def `should give a more clear error if properties or keys are inverted`(): Unit = {\n val musicDf = Seq(\n (1, 12, \"John Henry Bonham\", \"Drums\"),\n (2, 19, \"John Mayer\", \"Guitar\"),\n (3, 32, \"John Scofield\", \"Guitar\"),\n (4, 15, \"John Butler\", \"Guitar\")\n ).toDF(\"id\", \"experience\", \"name\", \"instrument\")\n\n try {\n musicDf.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"musician_name:name\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .save()\n } catch {\n case exception: IllegalArgumentException => {\n val clientException = ExceptionUtils.getRootCause(exception)\n assertTrue(clientException.getMessage.equals(\n \"\"\"Write failed due to the following errors:\n | - Schema is missing musician_name from option `relationship.source.node.keys`\n |\n |The option key and value might be inverted.\"\"\".stripMargin\n ))\n }\n case generic: Throwable =>\n fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}, got ${generic.getClass} instead\")\n }\n }\n\n @Test\n def `should give a more clear error if properties or keys are inverted on different options`(): Unit = {\n val musicDf = Seq(\n (1, 12, \"John Henry Bonham\", \"Drums\"),\n (2, 19, \"John Mayer\", \"Guitar\"),\n (3, 32, \"John Scofield\", \"Guitar\"),\n (4, 15, \"John Butler\", \"Guitar\")\n ).toDF(\"id\", \"experience\", \"name\", \"instrument\")\n\n try {\n musicDf.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"musician_name:name,another_name:name\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument_name:name\")\n .save()\n } catch {\n case exception: IllegalArgumentException => {\n val clientException = ExceptionUtils.getRootCause(exception)\n assertTrue(clientException.getMessage.equals(\n \"\"\"Write failed due to the following errors:\n | - Schema is missing instrument_name from option `relationship.target.node.keys`\n | - Schema is missing musician_name, another_name from option `relationship.source.node.keys`\n |\n |The option key and value might be inverted.\"\"\".stripMargin\n ))\n }\n case generic: Throwable =>\n fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}, got ${generic.getClass} instead\")\n }\n }\n\n @Test\n def `should give a more clear error if node properties or keys are inverted`(): Unit = {\n val musicDf = Seq(\n (1, 12, \"John Henry Bonham\", \"Drums\"),\n (2, 19, \"John Mayer\", \"Guitar\"),\n (3, 32, \"John Scofield\", \"Guitar\"),\n (4, 15, \"John Butler\", \"Guitar\")\n ).toDF(\"id\", \"experience\", \"name\", \"instrument\")\n\n try {\n musicDf.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"labels\", \"Person\")\n .option(\"node.properties\", \"musician_name:name,another_name:name\")\n .save()\n } catch {\n case exception: IllegalArgumentException => {\n val clientException = ExceptionUtils.getRootCause(exception)\n assertTrue(clientException.getMessage.equals(\n \"\"\"Write failed due to the following errors:\n | - Schema is missing instrument_name from option `node.properties`\n |\n |The option key and value might be inverted.\"\"\".stripMargin\n ))\n }\n case generic: Throwable => fail(\n s\"should be thrown a ${classOf[IllegalArgumentException].getName}, got ${generic.getClass} instead: ${generic.getMessage}\"\n )\n }\n }\n\n @Test\n def `exports write metrics`(): Unit = {\n val input = List(\"Ali\", \"Andrea\", \"Eugene\", \"Florent\")\n val query = \"CREATE (:Name {name: event.name})-[:STARTS_WITH]->(:Letter {value: left(event.name, 1)})\"\n val expectedMetrics = Map(\n DataWriterMetrics.RECORDS_WRITTEN_DESCRIPTION -> 4,\n DataWriterMetrics.RELATIONSHIPS_CREATED_DESCRIPTION -> 4,\n DataWriterMetrics.NODES_CREATED_DESCRIPTION -> 8,\n DataWriterMetrics.PROPERTIES_SET_DESCRIPTION -> 8\n )\n\n val metrics = new AtomicReference[Map[String, Any]]()\n val listener = new MetricsListener(expectedMetrics.keySet, metrics)\n ss.sparkContext.addSparkListener(listener)\n\n try {\n input.toDF(\"name\")\n .repartition(1)\n .write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"database\", \"db1\")\n .option(\"batch.size\", 1) // TODO: remove this when https://issues.apache.org/jira/browse/SPARK-45759 is fixed\n .option(\"query\", query)\n .save()\n\n val db1Session = SparkConnectorScalaSuiteIT.driver.session(SessionConfig.forDatabase(\"db1\"))\n Assert.assertEventually(\n () => {\n db1Session.run(\n \"MATCH (:Name)-[r:STARTS_WITH]->(:Letter) RETURN count(r) as cnt\"\n ).single().get(\"cnt\").asLong()\n },\n Matchers.equalTo(4L),\n 30L,\n TimeUnit.SECONDS\n )\n assertNotNull(metrics.get())\n assertEquals(metrics.get(), expectedMetrics)\n\n } finally {\n ss.sparkContext.removeSparkListener(listener)\n }\n }\n\n @Test\n def `does not create constraint if schema validation fails`(): Unit = {\n val cities = Seq(\n (1, \"Cherbourg en Cotentin\"),\n (2, \"London\"),\n (3, \"Malmö\")\n ).toDF(\"id\", \"city\")\n\n try {\n cities.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":News\")\n .option(\"node.keys\", \"newsId\")\n .option(\"schema.optimization.node.keys\", \"UNIQUE\")\n .save()\n } catch {\n case _: Exception => {}\n }\n\n var session: Session = null\n try {\n session = SparkConnectorScalaSuiteIT.driver.session()\n val result =\n session.run(\"SHOW CONSTRAINTS YIELD labelsOrTypes WHERE labelsOrTypes[0] = 'News' RETURN count(*) AS count\")\n .single()\n .get(\"count\")\n .asLong()\n assertEquals(0, result)\n } finally {\n if (session != null) {\n session.close()\n }\n }\n }\n\n class MetricsListener(names: Set[String], captureMetrics: AtomicReference[Map[String, Any]])\n extends SparkListener {\n\n override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = {\n stageSubmitted.stageInfo.accumulables.clear()\n }\n\n override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = {\n captureMetrics.set(stageCompleted\n .stageInfo\n .accumulables\n .values\n .filter(metric => metric.name.exists(names.contains))\n .map(metric => (metric.name.get, metric.value.get))\n .toMap)\n }\n }\n\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DataSourceWriterTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport junitparams.JUnitParamsRunner\nimport junitparams.Parameters\nimport org.apache.commons.lang3.exception.ExceptionUtils\nimport org.apache.spark.SparkException\nimport org.apache.spark.sql.DataFrame\nimport org.apache.spark.sql.SaveMode\nimport org.apache.spark.sql.SparkSession\nimport org.apache.spark.sql.types.ArrayType\nimport org.apache.spark.sql.types.ByteType\nimport org.apache.spark.sql.types.DataType\nimport org.apache.spark.sql.types.DataTypes\nimport org.apache.spark.sql.types.DayTimeIntervalType\nimport org.apache.spark.sql.types.DecimalType\nimport org.apache.spark.sql.types.YearMonthIntervalType\nimport org.junit\nimport org.junit.Assert._\nimport org.junit.Ignore\nimport org.junit.Test\nimport org.junit.runner.RunWith\nimport org.neo4j.driver.Result\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.Value\nimport org.neo4j.driver.exceptions.ClientException\nimport org.neo4j.driver.exceptions.value.Uncoercible\nimport org.neo4j.driver.internal.InternalPoint2D\nimport org.neo4j.driver.internal.InternalPoint3D\nimport org.neo4j.driver.internal.types.InternalTypeSystem\nimport org.neo4j.driver.summary.ResultSummary\nimport org.neo4j.driver.types.IsoDuration\nimport org.neo4j.driver.types.Type\nimport org.neo4j.spark.RowUtil.getByName\nimport org.neo4j.spark.util.Neo4jOptions\nimport org.scalatest.matchers.must.Matchers.be\nimport org.scalatest.matchers.must.Matchers.include\nimport org.scalatest.matchers.must.Matchers.the\nimport org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper\n\nimport java.time.LocalTime\nimport java.time.OffsetTime\nimport java.time.ZoneId\nimport java.time.ZoneOffset\nimport java.util.TimeZone\n\nimport scala.collection.JavaConverters._\nimport scala.collection.immutable.ListMap\nimport scala.language.postfixOps\nimport scala.math.Ordering.Implicits.infixOrderingOps\nimport scala.util.Random\n\nabstract class Neo4jType(`type`: String)\n\ncase class Duration(months: Long, days: Long, seconds: Long, nanoseconds: Long, `type`: String = \"duration\")\n extends Neo4jType(`type`)\n\ncase class Point2d(`type`: String = \"point-2d\", srid: Int, x: Double, y: Double) extends Neo4jType(`type`)\n\ncase class Point3d(`type`: String = \"point-3d\", srid: Int, x: Double, y: Double, z: Double) extends Neo4jType(`type`)\n\ncase class Time(`type`: String = \"offset-time\", value: String) extends Neo4jType(`type`)\n\ncase class LocalTimeValue(`type`: String = \"local-time\", value: String) extends Neo4jType(`type`)\n\ncase class Person(name: String, surname: String, age: Int, livesIn: Point3d)\n\ncase class Person_TimeAndLocalTime(name: String, time: Time, localTime: LocalTimeValue)\n\ncase class SimplePerson(name: String, surname: String)\n\ncase class EmptyRow[T](data: T)\n\n@RunWith(classOf[JUnitParamsRunner])\nclass DataSourceWriterTSE extends SparkConnectorScalaBaseTSE {\n\n val sparkSession = SparkSession.builder()\n .master(\"local[*]\")\n .appName(\"DataSourceWriterTSE\")\n .getOrCreate()\n\n import sparkSession.implicits._\n\n private def testType[T](ds: DataFrame, neo4jType: Type): Unit = {\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":MyNode:MyLabel\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:MyNode:MyLabel)\n |RETURN p.foo AS foo\n |\"\"\".stripMargin\n ).list().asScala\n .filter(r => r.get(\"foo\").hasType(neo4jType))\n .map(r => r.asMap().asScala)\n .toSet\n\n val expected = ds.collect()\n .map(row =>\n Map(\"foo\" -> {\n val foo = row.getAs[T](\"foo\")\n foo match {\n case sqlDate: java.sql.Date => sqlDate.toLocalDate\n case sqlTimestamp: java.sql.Timestamp => sqlTimestamp.toInstant.atZone(ZoneOffset.UTC)\n case _ => foo\n }\n })\n )\n .toSet\n\n assertEquals(expected, records)\n }\n\n private def testArray[T](ds: DataFrame): Unit = {\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":MyNode:MyLabel\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:MyNode:MyLabel)\n |RETURN p.foo AS foo\n |\"\"\".stripMargin\n ).list().asScala\n .filter(r => r.get(\"foo\").hasType(InternalTypeSystem.TYPE_SYSTEM.LIST()))\n .map(r => r.get(\"foo\").asList())\n .toSet\n val expected = ds.collect()\n .map(row => row.getList[T](0))\n .toSet\n\n assertEquals(expected, records)\n }\n\n private def testDurationType(ds: DataFrame, expected: Set[Duration]): Unit = {\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":Duration\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (d:Duration)\n |RETURN d.duration AS duration\n |\"\"\".stripMargin\n ).list().asScala\n .filter(r => r.get(\"duration\").hasType(InternalTypeSystem.TYPE_SYSTEM.DURATION()))\n .map(r => r.get(\"duration\").asIsoDuration())\n .map(data => Duration(data.months, data.days, data.seconds, data.nanoseconds))\n .toSet\n\n assertEquals(expected, records)\n }\n\n @Test\n def testThrowsExceptionIfNoValidReadOptionIsSet(): Unit = {\n try {\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .load()\n .show() // we need the action to be able to trigger the exception because of the changes in Spark 3\n } catch {\n case e: IllegalArgumentException =>\n assertEquals(\"No valid option found. One of `GDS`, `LABELS`, `QUERY`, `RELATIONSHIP` is required\", e.getMessage)\n case _: Throwable => fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n\n @Test\n def testThrowsExceptionIfTwoValidReadOptionAreSet(): Unit = {\n try {\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .option(\"relationship\", \"KNOWS\")\n .load() // we need the action to be able to trigger the exception because of the changes in Spark 3\n } catch {\n case e: IllegalArgumentException =>\n assertEquals(\n \"You need to specify just one of these options: 'gds', 'labels', 'query', 'relationship'\",\n e.getMessage\n )\n case _: Throwable => fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n\n @Test\n def testThrowsExceptionIfThreeValidReadOptionAreSet(): Unit = {\n try {\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .option(\"relationship\", \"KNOWS\")\n .option(\"query\", \"MATCH (n) RETURN n\")\n .load() // we need the action to be able to trigger the exception because of the changes in Spark 3\n } catch {\n case e: IllegalArgumentException =>\n assertEquals(\n \"You need to specify just one of these options: 'gds', 'labels', 'query', 'relationship'\",\n e.getMessage\n )\n case _: Throwable => fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n\n @Test\n def `should write nodes with string values into Neo4j`(): Unit = {\n val ds = (1 to 10)\n .map(i => i.toString)\n .toDF(\"foo\")\n\n testType[String](ds, InternalTypeSystem.TYPE_SYSTEM.STRING())\n }\n\n @Test\n def `should write nodes with string array values into Neo4j`(): Unit = {\n val ds = (1 to 10)\n .map(i => i.toString)\n .map(i => Array(i, i))\n .toDF(\"foo\")\n\n testArray[String](ds)\n }\n\n @Test\n def `should write nodes with int values into Neo4j`(): Unit = {\n val ds = (1 to 10)\n .map(i => i)\n .toDF(\"foo\")\n\n testType[Int](ds, InternalTypeSystem.TYPE_SYSTEM.INTEGER())\n }\n\n @Test\n def `should write nodes with byte values into Neo4j`(): Unit = {\n val ds = (1 to 10)\n .map(_.toByte)\n .toDF(\"foo\")\n\n testType[Byte](ds, InternalTypeSystem.TYPE_SYSTEM.INTEGER())\n }\n\n @Test\n def `should write nodes with short values into Neo4j`(): Unit = {\n val ds = (1 to 10)\n .map(_.toShort)\n .toDF(\"foo\")\n\n testType[Short](ds, InternalTypeSystem.TYPE_SYSTEM.INTEGER())\n }\n\n @Test\n def `should write nodes with date values into Neo4j`(): Unit = {\n val ds = (1 to 5)\n .map(i => java.sql.Date.valueOf(\"2020-01-0\" + i))\n .toDF(\"foo\")\n\n testType[java.sql.Date](ds, InternalTypeSystem.TYPE_SYSTEM.DATE())\n }\n\n @Test\n def `should write nodes with timestamp values into Neo4j`(): Unit = {\n val ds = (1 to 5)\n .map(i => java.sql.Timestamp.valueOf(s\"2020-01-0$i 11:11:11.11\"))\n .toDF(\"foo\")\n\n testType[java.sql.Timestamp](ds, InternalTypeSystem.TYPE_SYSTEM.DATE_TIME())\n }\n\n @Test\n def `should write nodes with timestampNTZ values into Neo4j`(): Unit = {\n val ds = (1 to 5)\n .map(i => java.time.LocalDateTime.of(2020, 1, i, 11, 11, 11, 111000000))\n .toDF(\"foo\")\n\n testType[java.time.LocalDateTime](ds, InternalTypeSystem.TYPE_SYSTEM.LOCAL_DATE_TIME())\n }\n\n @Test\n def `should write nodes with int array values into Neo4j`(): Unit = {\n val ds = (1 to 10)\n .map(i => i.toLong)\n .map(i => Array(i, i))\n .toDF(\"foo\")\n\n testArray[Long](ds)\n }\n\n @Test\n def `should write nodes with point-2d values into Neo4j`(): Unit = {\n val ds = (1 to 10)\n .map(i => EmptyRow(Point2d(srid = 4326, x = Random.nextDouble(), y = Random.nextDouble())))\n .toDS()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":MyNode:MyLabel\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:MyNode:MyLabel)\n |RETURN p.data AS data\n |\"\"\".stripMargin\n ).list().asScala\n .filter(r => r.get(\"data\").hasType(InternalTypeSystem.TYPE_SYSTEM.POINT()))\n .map(r => {\n val point = r.get(\"data\").asPoint()\n (point.srid(), point.x(), point.y())\n })\n .toSet\n val expected = ds.collect()\n .map(point => (point.data.srid, point.data.x, point.data.y))\n .toSet\n assertEquals(expected, records)\n }\n\n @Test\n def `should write nodes with point-2d array values into Neo4j`(): Unit = {\n val ds = (1 to 10)\n .map(i =>\n EmptyRow(Seq(\n Point2d(srid = 4326, x = Random.nextDouble(), y = Random.nextDouble()),\n Point2d(srid = 4326, x = Random.nextDouble(), y = Random.nextDouble())\n ))\n )\n .toDS()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":MyNode:MyLabel\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:MyNode:MyLabel)\n |RETURN p.data AS data\n |\"\"\".stripMargin\n ).list().asScala\n .filter(r => r.get(\"data\").hasType(InternalTypeSystem.TYPE_SYSTEM.LIST()))\n .map(r =>\n r.get(\"data\")\n .asList.asScala\n .map(_.asInstanceOf[InternalPoint2D])\n .map(point => (point.srid(), point.x(), point.y()))\n )\n .toSet\n val expected = ds.collect()\n .map(row => row.data.map(p => (p.srid, p.x, p.y)))\n .toSet\n assertEquals(expected, records)\n }\n\n @Test\n def `should write nodes with point-3d values into Neo4j`(): Unit = {\n val ds = (1 to 10)\n .map(i =>\n EmptyRow(Point3d(srid = 4979, x = Random.nextDouble(), y = Random.nextDouble(), z = Random.nextDouble()))\n )\n .toDS()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":MyNode:MyLabel\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:MyNode:MyLabel)\n |RETURN p.data AS data\n |\"\"\".stripMargin\n ).list().asScala\n .filter(r => r.get(\"data\").hasType(InternalTypeSystem.TYPE_SYSTEM.POINT()))\n .map(r => {\n val point = r.get(\"data\").asPoint()\n (point.srid(), point.x(), point.y())\n })\n .toSet\n val expected = ds.collect()\n .map(point => (point.data.srid, point.data.x, point.data.y))\n .toSet\n assertEquals(expected, records)\n }\n\n @Test\n def `should write nodes with point-3d array values into Neo4j`(): Unit = {\n val ds = (1 to 10)\n .map(i =>\n EmptyRow(Seq(\n Point3d(srid = 4979, x = Random.nextDouble(), y = Random.nextDouble(), z = Random.nextDouble()),\n Point3d(srid = 4979, x = Random.nextDouble(), y = Random.nextDouble(), z = Random.nextDouble())\n ))\n )\n .toDS()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":MyNode:MyLabel\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:MyNode:MyLabel)\n |RETURN p.data AS data\n |\"\"\".stripMargin\n ).list().asScala\n .filter(r => r.get(\"data\").hasType(InternalTypeSystem.TYPE_SYSTEM.LIST()))\n .map(r =>\n r.get(\"data\")\n .asList.asScala\n .map(_.asInstanceOf[InternalPoint3D])\n .map(point => (point.srid(), point.x(), point.y(), point.z()))\n )\n .toSet\n val expected = ds.collect()\n .map(row => row.data.map(p => (p.srid, p.x, p.y, p.z)))\n .toSet\n assertEquals(expected, records)\n }\n\n @Test\n def `should write nodes with map values into Neo4j`(): Unit = {\n val ds = (1 to 10)\n .map(i => Map(\"field\" + i -> i))\n .toDF(\"foo\")\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":MyNode:MyLabel\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:MyNode:MyLabel)\n |RETURN p\n |\"\"\".stripMargin\n ).list().asScala\n .filter(r => r.get(\"p\").hasType(InternalTypeSystem.TYPE_SYSTEM.MAP()))\n .map(r => r.get(\"p\").asMap().asScala)\n .toSet\n val expected = ds.collect().map(row => row.getMap[String, AnyRef](0))\n .map(map => map.map(t => (s\"foo.${t._1}\", t._2)).toMap)\n .toSet\n assertEquals(expected, records)\n }\n\n @Test\n def `should write nodes with duration values into Neo4j from java period`(): Unit = {\n val range = 1 to 10\n val ds = range\n .map(i => java.time.Period.ofMonths(i))\n .toDF(\"duration\")\n\n val expected = range\n .map(i => Duration(i, 0, 0, 0))\n .toSet\n\n testDurationType(ds, expected)\n }\n\n @Test\n def `should write nodes with duration values into Neo4j from java duration`(): Unit = {\n val range = 1 to 10\n val ds = range\n .map(i => java.time.Duration.ofDays(i.toLong))\n .toDF(\"duration\")\n\n val expected = range\n .map(i => Duration(0, i, 0, 0))\n .toSet\n\n testDurationType(ds, expected)\n }\n\n @Test\n def `should write nodes with duration values into Neo4j from struct`(): Unit = {\n val range = 1 to 10\n val ds = range\n .map(i => i.toLong)\n .map(i => EmptyRow(Duration(i, i, i, i)))\n .toDF(\"duration\")\n\n val expected = range\n .map(i => Duration(i, i, i, i))\n .toSet\n\n testDurationType(ds, expected)\n }\n\n @Test\n def `should write nodes with duration array values into Neo4j from struct`(): Unit = {\n val ds = (1 to 10)\n .map(i => i.toLong)\n .map(i =>\n EmptyRow(Seq(\n Duration(i, i, i, i),\n Duration(i, i, i, i)\n ))\n )\n .toDS()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"BeanWithDuration\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:BeanWithDuration)\n |RETURN p.data AS data\n |\"\"\".stripMargin\n ).list().asScala\n .map(r =>\n r.get(\"data\")\n .asList.asScala\n .map(_.asInstanceOf[IsoDuration])\n .map(data => (data.months, data.days, data.seconds, data.nanoseconds))\n )\n .toSet\n\n val expected = ds.collect()\n .map(row => row.data.map(data => (data.months, data.days, data.seconds, data.nanoseconds)))\n .toSet\n\n assertEquals(expected, records)\n }\n\n private case class DurationCase(\n intervalExpression: String,\n expectedDuration: Duration,\n expectedDt: Class[_ <: DataType] = classOf[DayTimeIntervalType]\n ) {\n private val isArithmetic = intervalExpression.startsWith(\"timestamp\")\n\n val sql: String = if (isArithmetic) {\n intervalExpression\n } else {\n s\"INTERVAL $intervalExpression\"\n }\n }\n\n private def sqlDurationCases: java.util.List[DurationCase] = java.util.Arrays.asList(\n // DAY/TIME -> DayTimeIntervalType\n DurationCase(\"'3' DAY\", Duration(0, 3, 0, 0)),\n DurationCase(\"'10 05' DAY TO HOUR\", Duration(0, 10, 5L * 3600, 0)),\n DurationCase(\"'10 05:30' DAY TO MINUTE\", Duration(0, 10, 5L * 3600 + 30L * 60, 0)),\n DurationCase(\"'10 05:30:15.123456' DAY TO SECOND\", Duration(0, 10, 5L * 3600 + 30L * 60 + 15L, 123456000)),\n DurationCase(\"'12' HOUR\", Duration(0, 0, 12L * 3600, 0)),\n DurationCase(\"'12:34' HOUR TO MINUTE\", Duration(0, 0, 12L * 3600 + 34L * 60, 0)),\n DurationCase(\"'12:34:56.123456' HOUR TO SECOND\", Duration(0, 0, 12L * 3600 + 34L * 60 + 56L, 123456000)),\n DurationCase(\"'42' MINUTE\", Duration(0, 0, 42L * 60, 0)),\n DurationCase(\"'42:07.001002' MINUTE TO SECOND\", Duration(0, 0, 42L * 60 + 7L, 1002000)),\n DurationCase(\"'59.000001' SECOND\", Duration(0, 0, 59L, 1000)),\n DurationCase(\n \"timestamp('2025-01-02 18:30:00.454') - timestamp('2024-01-01 00:00:00')\",\n Duration(0, 367, 66600L, 454000000)\n ),\n // YEAR/MONTH -> YearMonthIntervalType\n DurationCase(\"'3' YEAR\", Duration(36, 0, 0, 0), classOf[YearMonthIntervalType]),\n DurationCase(\"'7' MONTH\", Duration(7, 0, 0, 0), classOf[YearMonthIntervalType]),\n DurationCase(\"'4-5' YEAR TO MONTH\", Duration(53, 0, 0, 0), classOf[YearMonthIntervalType])\n )\n\n @Test\n @Parameters(method = \"sqlDurationCases\")\n def `interval SQL literals map to native neo4j durations`(testCase: DurationCase): Unit = {\n val id = java.util.UUID.randomUUID().toString\n val df = sparkSession.sql(s\"SELECT '$id' AS id, ${testCase.sql} AS duration\")\n\n df.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Dur\")\n .save()\n\n val wantType = testCase.expectedDt.getSimpleName\n val gotType = df.schema(\"duration\").dataType\n assertTrue(s\"expected Spark to pick $wantType but it was $gotType\", testCase.expectedDt.isInstance(gotType))\n\n val gotDuration = SparkConnectorScalaSuiteIT.session().run(\n s\"\"\"MATCH (d:Dur {id: '$id'})\n |RETURN d.duration AS duration\n |\"\"\".stripMargin\n ).single().get(\"duration\").asIsoDuration()\n\n assertEquals(s\"${testCase.sql} -> months\", testCase.expectedDuration.months, gotDuration.months)\n assertEquals(s\"${testCase.sql} -> days\", testCase.expectedDuration.days, gotDuration.days)\n assertEquals(s\"${testCase.sql} -> seconds\", testCase.expectedDuration.seconds, gotDuration.seconds)\n assertEquals(s\"${testCase.sql} -> nanos\", testCase.expectedDuration.nanoseconds, gotDuration.nanoseconds)\n }\n\n private val sqlDurationArrayCases: java.util.List[Seq[DurationCase]] = java.util.Arrays.asList(\n Seq(\n DurationCase(\"'10 05:30:15.123' DAY TO SECOND\", null),\n DurationCase(\"'0 00:00:01.000' DAY TO SECOND\", null)\n ),\n Seq(\n DurationCase(\"timestamp('2024-01-02 00:00:00') - timestamp('2024-01-01 00:00:00')\", null),\n DurationCase(\"timestamp('2024-01-01 00:00:00') - current_timestamp()\", null)\n ),\n Seq(\n DurationCase(\"'1-02' YEAR TO MONTH\", null, classOf[YearMonthIntervalType]),\n DurationCase(\"'0-11' YEAR TO MONTH\", null, classOf[YearMonthIntervalType])\n )\n )\n\n @Test\n @Parameters(method = \"sqlDurationArrayCases\")\n def `interval SQL arrays map to native neo4j durations arrays`(testCase: Seq[DurationCase]): Unit = {\n val id = java.util.UUID.randomUUID().toString\n val expectedDt = testCase.head.expectedDt\n val sqlArray = testCase.map(_.sql).mkString(\"array(\", \", \", \")\")\n val df = sparkSession.sql(s\"SELECT '$id' AS id, $sqlArray AS durations\")\n\n df.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"DurArr\")\n .save()\n\n val gotType = df.schema(\"durations\").dataType\n\n assertTrue(\n s\"expected Spark to infer ArrayType(${expectedDt.getSimpleName}) but it was $gotType\",\n gotType match {\n case ArrayType(et, _) if expectedDt.isInstance(et) => true\n case _ => false\n }\n )\n\n val result = SparkConnectorScalaSuiteIT.session().run(\n s\"\"\"MATCH (d:DurArr {id: '$id'})\n |RETURN d.durations AS durations\n |\"\"\".stripMargin\n ).single().get(\"durations\")\n\n assertTrue(\n s\"expected successful conversion to IsoDuration array, but it failed: $result\",\n try {\n val _ = result.asList((v: Value) => v.asIsoDuration())\n true\n } catch {\n case _: Uncoercible => false\n case e => throw e\n }\n )\n }\n\n @Test\n def `should write TINYINT as neo4j integer`(): Unit = {\n val id = java.util.UUID.randomUUID().toString\n val df = sparkSession.sql(s\"SELECT '$id' AS id, CAST(5 AS TINYINT) AS byte\")\n\n df.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Byte\")\n .save()\n\n val wantType = DataTypes.ByteType\n val gotType = df.schema(\"byte\").dataType\n assertTrue(s\"expected Spark to pick ${wantType.simpleString} but it was $gotType\", wantType == gotType)\n\n val gotByte = SparkConnectorScalaSuiteIT.session().run(\n s\"\"\"MATCH (b:Byte {id: '$id'})\n |RETURN b.byte AS byte\n |\"\"\".stripMargin\n ).single().get(\"byte\").asInt()\n\n assertEquals(5, gotByte)\n }\n\n @Test\n def `should write BINARY (byte array) as neo4j ByteArray`(): Unit = {\n val id = java.util.UUID.randomUUID().toString\n val sqlArray = (1 to 10).map(i => s\"CAST($i AS TINYINT)\").mkString(\"array(\", \", \", \")\")\n val df = sparkSession.sql(s\"SELECT '$id' AS id, $sqlArray AS binary\")\n\n df.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Binary\")\n .save()\n\n val wantType: DataType = DataTypes.ByteType\n val gotType = df.schema(\"binary\").dataType\n\n assertTrue(\n s\"expected Spark to infer ArrayType(${wantType.simpleString}) but it was $gotType\",\n gotType match {\n case ArrayType(_: ByteType, _) => true\n case _ => false\n }\n )\n\n val gotByteArray = SparkConnectorScalaSuiteIT.session().run(\n s\"\"\"MATCH (b:Binary {id: '$id'})\n |RETURN b.binary AS binary\n |\"\"\".stripMargin\n ).single().get(\"binary\").asByteArray()\n\n assertEquals(10, gotByteArray.length)\n\n for (b <- gotByteArray.indices) {\n val expectedValue = (b + 1).toByte\n assertEquals(expectedValue, gotByteArray(b))\n }\n }\n\n @Test\n def `should write SMALLINT as neo4j integer`(): Unit = {\n val id = java.util.UUID.randomUUID().toString\n val df = sparkSession.sql(s\"SELECT '$id' AS id, CAST(5 AS SMALLINT) AS short\")\n\n df.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Short\")\n .save()\n\n val wantType = DataTypes.ShortType\n val gotType = df.schema(\"short\").dataType\n assertTrue(s\"expected Spark to pick ${wantType.simpleString} but it was $gotType\", wantType == gotType)\n\n val gotByte = SparkConnectorScalaSuiteIT.session().run(\n s\"\"\"MATCH (b:Short {id: '$id'})\n |RETURN b.short AS short\n |\"\"\".stripMargin\n ).single().get(\"short\").asInt()\n\n assertEquals(5, gotByte)\n }\n\n @Test\n def `should write DECIMAL as neo4j string`(): Unit = {\n val id = java.util.UUID.randomUUID().toString\n val df = sparkSession.sql(s\"SELECT '$id' AS id, CAST(5.42 AS DECIMAL(10, 2)) AS decimal\")\n\n df.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Decimal\")\n .save()\n\n val wantType = DecimalType(10, 2)\n val gotType = df.schema(\"decimal\").dataType\n assertTrue(s\"expected Spark to pick $wantType but it was $gotType\", wantType.typeName == gotType.simpleString)\n\n val gotDecimal = SparkConnectorScalaSuiteIT.session().run(\n s\"\"\"MATCH (b:Decimal {id: '$id'})\n |RETURN b.decimal AS decimal\n |\"\"\".stripMargin\n ).single().get(\"decimal\").asString\n\n assertEquals(\"5.42\", gotDecimal)\n }\n\n @Test\n def `should write nodes into Neo4j with points`(): Unit = {\n val total = 10\n val rand = Random\n val ds = (1 to total)\n .map(i =>\n Person(\n name = \"Andrea \" + i,\n \"Santurbano \" + i,\n rand.nextInt(100),\n Point3d(srid = 4979, x = 12.5811776, y = 41.9579492, z = 1.3)\n )\n ).toDS()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":Person: Customer\")\n .save()\n\n val count = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:Person:Customer)\n |WHERE p.name STARTS WITH 'Andrea'\n |AND p.surname STARTS WITH 'Santurbano'\n |RETURN count(p) AS count\n |\"\"\".stripMargin\n ).single().get(\"count\").asInt()\n assertEquals(total, count)\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:Person:Customer)\n |WHERE p.name STARTS WITH 'Andrea'\n |AND p.surname STARTS WITH 'Santurbano'\n |RETURN p.name AS name, p.surname AS surname, p.age AS age,\n | p.bornIn AS bornIn, p.livesIn AS livesIn\n |\"\"\".stripMargin\n ).list().asScala\n .filter(r => {\n val map: java.util.Map[String, Object] = r.asMap()\n (map.get(\"name\").isInstanceOf[String]\n && map.get(\"surname\").isInstanceOf[String]\n && map.get(\"livesIn\").isInstanceOf[InternalPoint3D]\n && map.get(\"age\").isInstanceOf[Long])\n })\n assertEquals(total, records.size)\n }\n\n @Test\n def `should write nodes into Neo4j with Time and LocalTime Types`(): Unit = {\n val total = 1\n val rand = Random\n val ds = (1 to total)\n .map(i =>\n Person_TimeAndLocalTime(\n name = \"Andrea\",\n time = Time(value = \"12:50:35.556000000+01:00\"),\n localTime = LocalTimeValue(value = \"12:50:35.556000000\")\n )\n ).toDS()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"node.keys\", \"name\")\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":Person_TimeAndLocalTime\")\n .save()\n\n val count = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:Person_TimeAndLocalTime)\n |WHERE p.name STARTS WITH 'Andrea' \n |RETURN count(p) AS count\n |\"\"\".stripMargin\n ).single().get(\"count\").asInt()\n assertEquals(total, count)\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:Person_TimeAndLocalTime)\n |WHERE p.name STARTS WITH 'Andrea'\n |RETURN p.name AS name, p.time AS time, p.localTime AS localTime\n |\"\"\".stripMargin\n ).list().asScala\n .filter(r => {\n val map: java.util.Map[String, Object] = r.asMap()\n (map.get(\"name\").isInstanceOf[String]\n && map.get(\"time\").isInstanceOf[OffsetTime]\n && map.get(\"localTime\").isInstanceOf[LocalTime])\n })\n assertEquals(total, records.size)\n }\n\n @Test\n def `should throw an error because the node already exists`(): Unit = {\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(new TransactionWork[Result] {\n override def execute(transaction: Transaction): Result =\n transaction.run(\"CREATE CONSTRAINT person_surname FOR (p:Person) REQUIRE p.surname IS UNIQUE\")\n })\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(new TransactionWork[Result] {\n override def execute(transaction: Transaction): Result =\n transaction.run(\"CREATE (p:Person{name: 'Andrea', surname: 'Santurbano'})\")\n })\n\n val ds = Seq(SimplePerson(\"Andrea\", \"Santurbano\")).toDS()\n\n try {\n val thrown = the[SparkException] thrownBy {\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .save() // we need the action to be able to trigger the exception because of the changes in Spark 3\n }\n\n thrown.getMessage should include(\"org.neo4j.driver.exceptions.ClientException\")\n val rootCause = ExceptionUtils.getRootCause(thrown)\n // root cause is not always returned as a ClientException so we pass it through pattern matching to remove flakiness\n rootCause match {\n case c: ClientException =>\n c.code() should be(\"Neo.ClientError.Schema.ConstraintValidationFailed\")\n case _ =>\n }\n } finally {\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(new TransactionWork[Result] {\n override def execute(transaction: Transaction): Result = transaction.run(\"DROP CONSTRAINT person_surname\")\n })\n }\n }\n\n @Test\n def `should update the node that already exists`(): Unit = {\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(new TransactionWork[Result] {\n override def execute(transaction: Transaction): Result =\n transaction.run(\"CREATE CONSTRAINT person_surname FOR (p:Person) REQUIRE p.surname IS UNIQUE\")\n })\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(new TransactionWork[Result] {\n override def execute(transaction: Transaction): Result =\n transaction.run(\"CREATE (p:Person{name: 'Federico', surname: 'Santurbano'})\")\n })\n\n val ds = Seq(SimplePerson(\"Andrea\", \"Santurbano\")).toDS()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .option(\"node.keys\", \"surname\")\n .save()\n\n val nodeList = SparkConnectorScalaSuiteIT.session()\n .run(\n \"\"\"MATCH (n:Person{surname: 'Santurbano'})\n |RETURN n\n |\"\"\".stripMargin\n )\n .list()\n .asScala\n assertEquals(1, nodeList.size)\n assertEquals(\"Andrea\", nodeList.head.get(\"n\").asNode().get(\"name\").asString())\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(new TransactionWork[Result] {\n override def execute(transaction: Transaction): Result = transaction.run(\"DROP CONSTRAINT person_surname\")\n })\n }\n\n @Test\n def `should skip null properties`(): Unit = {\n val ds = Seq(SimplePerson(\"Andrea\", null)).toDS()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .save()\n\n val nodeList = SparkConnectorScalaSuiteIT.session()\n .run(\n \"\"\"MATCH (n:Person{name: 'Andrea'})\n |RETURN n\n |\"\"\".stripMargin\n )\n .list()\n .asScala\n assertEquals(1, nodeList.size)\n val node = nodeList.head.get(\"n\").asNode()\n assertFalse(\"surname should not exist\", node.asMap().containsKey(\"surname\"))\n }\n\n @Test\n def `should throw an error because SaveMode.Overwrite need node.keys`(): Unit = {\n val ds = Seq(SimplePerson(\"Andrea\", \"Santurbano\")).toDS()\n try {\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \"Person\")\n .save() // we need the action to be able to trigger the exception because of the changes in Spark 3\n } catch {\n case illegalArgumentException: IllegalArgumentException => {\n assertTrue(illegalArgumentException.getMessage.equals(\n s\"${Neo4jOptions.NODE_KEYS} is required when Save Mode is Overwrite\"\n ))\n }\n case e: Throwable =>\n fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName} but is ${e.getClass.getSimpleName}\")\n }\n }\n\n @Test\n def `should write within partitions`(): Unit = {\n val ds = (1 to 100).map(i => Person(\"Andrea \" + i, \"Santurbano \" + i, 36, null)).toDS()\n .repartition(10)\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":Person:Customer\")\n .option(\"batch.size\", \"11\")\n .save()\n\n val count = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (p:Person:Customer)\n |WHERE p.name STARTS WITH 'Andrea'\n |AND p.surname STARTS WITH 'Santurbano'\n |RETURN count(p) AS count\n |\"\"\".stripMargin\n ).single().get(\"count\").asInt()\n assertEquals(100, count)\n\n val keys = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (p:Person:Customer)\n |WHERE p.name STARTS WITH 'Andrea'\n |AND p.surname STARTS WITH 'Santurbano'\n |RETURN DISTINCT keys(p) AS keys\n |\"\"\".stripMargin\n ).single().get(\"keys\").asList()\n assertEquals(Set(\"name\", \"surname\", \"age\"), keys.asScala.toSet)\n }\n\n @Test\n @Ignore(\"This won't work right now because we can't know if we are in a Write or Read context\")\n def `should throw an exception for a read only query`(): Unit = {\n val ds = (1 to 100).map(i => Person(\"Andrea \" + i, \"Santurbano \" + i, 36, null)).toDS()\n\n try {\n ds.write\n .mode(SaveMode.Overwrite)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"MATCH (r:Read) RETURN r\")\n .option(\"batch.size\", \"11\")\n .save() // we need the action to be able to trigger the exception because of the changes in Spark 3\n } catch {\n case illegalArgumentException: IllegalArgumentException =>\n assertTrue(illegalArgumentException.getMessage.equals(\"Please provide a valid WRITE query\"))\n case t: Throwable => fail(\n s\"should be thrown a ${classOf[IllegalArgumentException].getName}, but it's ${t.getClass.getSimpleName}: ${t.getMessage}\"\n )\n }\n }\n\n @Test\n def `should insert data with a custom query`(): Unit = {\n val ds = (1 to 100).map(i => Person(\"Andrea \" + i, \"Santurbano \" + i, 36, null)).toDS()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"CREATE (n:MyNode{fullName: event.name + event.surname, age: event.age - 10})\")\n .option(\"batch.size\", \"11\")\n .save()\n\n val count = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (p:MyNode)\n |WHERE p.fullName CONTAINS 'Andrea'\n |AND p.fullName CONTAINS 'Santurbano'\n |AND p.age = 26\n |RETURN count(p) AS count\n |\"\"\".stripMargin\n ).single().get(\"count\").asLong()\n assertEquals(ds.count(), count)\n }\n\n @Test\n def `should handle unusual column names`(): Unit = {\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(new TransactionWork[Result] {\n override def execute(transaction: Transaction): Result =\n transaction.run(\"CREATE CONSTRAINT instrument_name FOR (i:Instrument) REQUIRE i.name IS UNIQUE\")\n })\n\n val musicDf = Seq(\n (12, \"John Bonham\", \"Drums\", \"f``````oo\"),\n (19, \"John Mayer\", \"Guitar\", \"bar\"),\n (32, \"John Scofield\", \"Guitar\", \"ba` z\"),\n (15, \"John Butler\", \"Guitar\", \"qu ux\")\n ).toDF(\"experience\", \"name\", \"instrument\", \"fi``(╯°□°)╯︵ ┻━┻eld\")\n\n musicDf.write\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.source.node.keys\", \"name\")\n .option(\"relationship.source.node.properties\", \"fi``(╯°□°)╯︵ ┻━┻eld:field\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .save()\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(new TransactionWork[Result] {\n override def execute(transaction: Transaction): Result = transaction.run(\"DROP CONSTRAINT instrument_name\")\n })\n\n val musicDfCheck = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .load()\n\n val size = musicDfCheck.count\n assertEquals(4, size)\n\n val res = musicDfCheck.orderBy(\"`source.name`\").collectAsList()\n\n assertEquals(\"John Bonham\", res.get(0).getString(4))\n assertEquals(\"f``````oo\", res.get(0).getString(5))\n assertEquals(\"Drums\", res.get(0).getString(8))\n\n assertEquals(\"John Butler\", res.get(1).getString(4))\n assertEquals(\"qu ux\", res.get(1).getString(5))\n assertEquals(\"Guitar\", res.get(1).getString(8))\n\n assertEquals(\"John Mayer\", res.get(2).getString(4))\n assertEquals(\"bar\", res.get(2).getString(5))\n assertEquals(\"Guitar\", res.get(2).getString(8))\n\n assertEquals(\"John Scofield\", res.get(3).getString(4))\n assertEquals(\"ba` z\", res.get(3).getString(5))\n assertEquals(\"Guitar\", res.get(3).getString(8))\n }\n\n @Test(expected = classOf[SparkException])\n def `should give error if native mode doesn't find a valid schema`(): Unit = {\n val musicDf = Seq(\n (12, \"John Bonham\", \"Drums\"),\n (19, \"John Mayer\", \"Guitar\"),\n (32, \"John Scofield\", \"Guitar\"),\n (15, \"John Butler\", \"Guitar\")\n ).toDF(\"experience\", \"name\", \"instrument\")\n\n try {\n musicDf.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.save.strategy\", \"NATIVE\")\n .option(\"relationship.source.labels\", \":Person\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .save() // we need the action to be able to trigger the exception because of the changes in Spark 3\n } catch {\n case sparkException: SparkException => {\n val clientException = ExceptionUtils.getRootCause(sparkException)\n assertTrue(clientException.getMessage.equals(\n \"NATIVE write strategy requires a schema like: rel.[props], source.[props], target.[props]. \" +\n \"All of these columns are empty in the current schema.\"\n ))\n throw sparkException\n }\n case _: Throwable => fail(s\"should be thrown a ${classOf[SparkException].getName}\")\n }\n }\n\n @Test\n def `should write relations with KEYS mode`(): Unit = {\n val musicDf = Seq(\n (12, \"John Bonham\", \"Drums\"),\n (19, \"John Mayer\", \"Guitar\"),\n (32, \"John Scofield\", \"Guitar\"),\n (15, \"John Butler\", \"Guitar\")\n ).toDF(\"experience\", \"name\", \"instrument\")\n\n musicDf.repartition(1).write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"name:name\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .save()\n\n val df2 = ss.read.format(classOf[DataSource].getName)\n .option(\"batch.size\", 100)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .load()\n\n assertEquals(4, df2.count())\n\n val res = df2.orderBy(\"`source.name`\").collectAsList()\n\n assertEquals(\"John Bonham\", res.get(0).getString(4))\n assertEquals(\"Drums\", res.get(0).getString(7))\n\n assertEquals(\"John Butler\", res.get(1).getString(4))\n assertEquals(\"Guitar\", res.get(1).getString(7))\n\n assertEquals(\"John Mayer\", res.get(2).getString(4))\n assertEquals(\"Guitar\", res.get(2).getString(7))\n\n assertEquals(\"John Scofield\", res.get(3).getString(4))\n assertEquals(\"Guitar\", res.get(3).getString(7))\n }\n\n @Test\n def `should fail validating options if ErrorIfExists is used`(): Unit = {\n val musicDf = Seq(\n (12, \"John Bonham\", \"Drums\"),\n (19, \"John Mayer\", \"Guitar\"),\n (32, \"John Scofield\", \"Guitar\"),\n (15, \"John Butler\", \"Guitar\")\n ).toDF(\"experience\", \"name\", \"instrument\")\n\n try {\n musicDf.repartition(1).write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.save.mode\", \"ErrorIfExists\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"name:name\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .save()\n } catch {\n case e: IllegalArgumentException =>\n assertEquals(\"Save mode 'ErrorIfExists' is not supported on Spark 3.0, use 'Append' instead.\", e.getMessage)\n case _: Throwable => fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n\n @Test\n @Ignore(\"trying to recreate the deadlock issue\")\n def `should give better errors if transaction fails`(): Unit = {\n val df = List.fill(200)((\"John Bonham\", \"Drums\")).toDF(\"name\", \"instrument\")\n\n df.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"name:name\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .save()\n\n df.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"transaction.retries\", 0)\n .option(\"partitions\", \"10\")\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"name:name\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .save()\n }\n\n def writeKeyModeRelationshipWriteDataSet(\n optionModifier: Map[String, String] => Map[String, String] = { m => m }\n ): DataFrame = {\n val musicDf = Seq(\n (12, \"John Bonham\", \"Drums\", 2, true),\n (19, \"John Mayer\", \"Guitar\", 1, false),\n (32, \"John Scofield\", \"Guitar\", 3, true),\n (15, \"John Butler\", \"Guitar\", 4, false)\n ).toDF(\"experience\", \"name\", \"instrument\", \"rating\", \"hasDiploma\")\n\n val options = Map(\n \"url\" -> SparkConnectorScalaSuiteIT.server.getBoltUrl,\n \"relationship\" -> \"PLAYS\",\n \"relationship.source.save.mode\" -> \"Overwrite\",\n \"relationship.target.save.mode\" -> \"Overwrite\",\n \"relationship.save.strategy\" -> \"keys\",\n \"relationship.source.labels\" -> \":Musician\",\n \"relationship.source.node.keys\" -> \"name\",\n \"relationship.target.labels\" -> \":Instrument\",\n \"relationship.target.node.keys\" -> \"instrument:name\"\n )\n val modifiedOptions = optionModifier(options)\n\n musicDf.repartition(1).write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .options(modifiedOptions)\n .save()\n\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .load()\n }\n\n @Test\n def `should write relations with KEYS mode with explicitly listed properties`(): Unit = {\n val resultDf = writeKeyModeRelationshipWriteDataSet({ options =>\n options + (\"relationship.properties\" -> \"experience, rating:avgRating, instrument\")\n })\n\n resultDf.show(false)\n assertEquals(4, resultDf.count())\n\n val res = resultDf.orderBy(\"`source.name`\").collectAsList()\n\n assertEquals(\"John Bonham\", getByName[String](res.get(0), \"source.name\"))\n assertEquals(\"Drums\", getByName[String](res.get(0), \"target.name\"))\n assertEquals(\"Drums\", getByName[String](res.get(0), \"rel.instrument\"))\n assertEquals(12, getByName[Long](res.get(0), \"rel.experience\"))\n assertEquals(2, getByName[Long](res.get(0), \"rel.avgRating\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have hasDiploma field\",\n res.get(0).fieldIndex(\"rel.hasDiploma\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have rating field\",\n res.get(0).fieldIndex(\"rel.rating\")\n )\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(0).fieldIndex(\"rel.name\"))\n\n assertEquals(\"John Butler\", getByName[String](res.get(1), \"source.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(1), \"target.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(1), \"rel.instrument\"))\n assertEquals(15, getByName[Long](res.get(1), \"rel.experience\"))\n assertEquals(4, getByName[Long](res.get(1), \"rel.avgRating\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have hasDiploma field\",\n res.get(1).fieldIndex(\"rel.hasDiploma\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have rating field\",\n res.get(1).fieldIndex(\"rel.rating\")\n )\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(1).fieldIndex(\"rel.name\"))\n\n assertEquals(\"John Mayer\", getByName[String](res.get(2), \"source.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(2), \"target.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(2), \"rel.instrument\"))\n assertEquals(19, getByName[Long](res.get(2), \"rel.experience\"))\n assertEquals(1, getByName[Long](res.get(2), \"rel.avgRating\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have hasDiploma field\",\n res.get(2).fieldIndex(\"rel.hasDiploma\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have rating field\",\n res.get(2).fieldIndex(\"rel.rating\")\n )\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(2).fieldIndex(\"rel.name\"))\n\n assertEquals(\"John Scofield\", getByName[String](res.get(3), \"source.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(3), \"target.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(3), \"rel.instrument\"))\n assertEquals(32, getByName[Long](res.get(3), \"rel.experience\"))\n assertEquals(3, getByName[Long](res.get(3), \"rel.avgRating\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have hasDiploma field\",\n res.get(3).fieldIndex(\"rel.hasDiploma\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have rating field\",\n res.get(3).fieldIndex(\"rel.rating\")\n )\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(3).fieldIndex(\"rel.name\"))\n }\n\n @Test\n def `should write relations with KEYS mode with explicitly listed empty properties`(): Unit = {\n val resultDf = writeKeyModeRelationshipWriteDataSet({ options =>\n options + (\"relationship.properties\" -> \"\")\n })\n\n resultDf.show(false)\n assertEquals(4, resultDf.count())\n\n val res = resultDf.orderBy(\"`source.name`\").collectAsList()\n\n assertEquals(\"John Bonham\", getByName[String](res.get(0), \"source.name\"))\n assertEquals(\"Drums\", getByName[String](res.get(0), \"target.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have experience field\",\n res.get(0).fieldIndex(\"rel.experience\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have hasDiploma field\",\n res.get(0).fieldIndex(\"rel.hasDiploma\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have rating field\",\n res.get(0).fieldIndex(\"rel.rating\")\n )\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(0).fieldIndex(\"rel.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have instrument field\",\n res.get(0).fieldIndex(\"rel.instrument\")\n )\n\n assertEquals(\"John Butler\", getByName[String](res.get(1), \"source.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(1), \"target.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have experience field\",\n res.get(1).fieldIndex(\"rel.experience\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have hasDiploma field\",\n res.get(1).fieldIndex(\"rel.hasDiploma\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have rating field\",\n res.get(1).fieldIndex(\"rel.rating\")\n )\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(1).fieldIndex(\"rel.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have instrument field\",\n res.get(1).fieldIndex(\"rel.instrument\")\n )\n\n assertEquals(\"John Mayer\", getByName[String](res.get(2), \"source.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(2), \"target.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have experience field\",\n res.get(2).fieldIndex(\"rel.experience\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have hasDiploma field\",\n res.get(2).fieldIndex(\"rel.hasDiploma\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have rating field\",\n res.get(2).fieldIndex(\"rel.rating\")\n )\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(2).fieldIndex(\"rel.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have instrument field\",\n res.get(2).fieldIndex(\"rel.instrument\")\n )\n\n assertEquals(\"John Scofield\", getByName[String](res.get(3), \"source.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(3), \"target.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have experience field\",\n res.get(3).fieldIndex(\"rel.experience\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have hasDiploma field\",\n res.get(3).fieldIndex(\"rel.hasDiploma\")\n )\n assertThrows[IllegalArgumentException](\n \"relationship should not have rating field\",\n res.get(3).fieldIndex(\"rel.rating\")\n )\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(3).fieldIndex(\"rel.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have instrument field\",\n res.get(3).fieldIndex(\"rel.instrument\")\n )\n }\n\n @Test\n def `should write relations with KEYS mode with default properties`(): Unit = {\n val resultDf = writeKeyModeRelationshipWriteDataSet()\n\n resultDf.show(false)\n assertEquals(4, resultDf.count())\n\n val res = resultDf.orderBy(\"`source.name`\").collectAsList()\n\n assertEquals(\"John Bonham\", getByName[String](res.get(0), \"source.name\"))\n assertEquals(\"Drums\", getByName[String](res.get(0), \"target.name\"))\n assertEquals(12, getByName[Long](res.get(0), \"rel.experience\"))\n assertEquals(true, getByName[Boolean](res.get(0), \"rel.hasDiploma\"))\n assertEquals(2, getByName[Long](res.get(0), \"rel.rating\"))\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(0).fieldIndex(\"rel.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have instrument field\",\n res.get(0).fieldIndex(\"rel.instrument\")\n )\n\n assertEquals(\"John Butler\", getByName[String](res.get(1), \"source.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(1), \"target.name\"))\n assertEquals(15, getByName[Long](res.get(1), \"rel.experience\"))\n assertEquals(false, getByName[Boolean](res.get(1), \"rel.hasDiploma\"))\n assertEquals(4, getByName[Long](res.get(1), \"rel.rating\"))\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(1).fieldIndex(\"rel.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have instrument field\",\n res.get(1).fieldIndex(\"rel.instrument\")\n )\n\n assertEquals(\"John Mayer\", getByName[String](res.get(2), \"source.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(2), \"target.name\"))\n assertEquals(19, getByName[Long](res.get(2), \"rel.experience\"))\n assertEquals(false, getByName[Boolean](res.get(2), \"rel.hasDiploma\"))\n assertEquals(1, getByName[Long](res.get(2), \"rel.rating\"))\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(2).fieldIndex(\"rel.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have instrument field\",\n res.get(2).fieldIndex(\"rel.instrument\")\n )\n\n assertEquals(\"John Scofield\", getByName[String](res.get(3), \"source.name\"))\n assertEquals(\"Guitar\", getByName[String](res.get(3), \"target.name\"))\n assertEquals(32, getByName[Long](res.get(3), \"rel.experience\"))\n assertEquals(true, getByName[Boolean](res.get(3), \"rel.hasDiploma\"))\n assertEquals(3, getByName[Long](res.get(3), \"rel.rating\"))\n assertThrows[IllegalArgumentException](\"relationship should not have name field\", res.get(3).fieldIndex(\"rel.name\"))\n assertThrows[IllegalArgumentException](\n \"relationship should not have instrument field\",\n res.get(3).fieldIndex(\"rel.instrument\")\n )\n }\n\n @Test\n def `should read and write relations with node overwrite mode`(): Unit = {\n val fixtureQuery: String =\n s\"\"\"CREATE (m:Musician {id: 1, name: \"John Bonham\"})\n |CREATE (i:Instrument {name: \"Drums\"})\n |CREATE (m)-[:PLAYS {experience: 10}]->(i)\n |RETURN *\n \"\"\".stripMargin\n\n SparkConnectorScalaSuiteIT.driver.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(fixtureQuery).consume()\n }\n )\n\n val musicDf = Seq(\n (1, 12, \"John Henry Bonham\", \"Drums\"),\n (2, 19, \"John Mayer\", \"Guitar\"),\n (3, 32, \"John Scofield\", \"Guitar\"),\n (4, 15, \"John Butler\", \"Guitar\")\n ).toDF(\"id\", \"experience\", \"name\", \"instrument\")\n\n musicDf.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.properties\", \"experience\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"id\")\n .option(\"relationship.source.node.properties\", \"name\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .save()\n\n val df2 = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship.nodes.map\", \"false\")\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .load()\n\n val result = df2.where(\"`source.id` = 1\")\n .collectAsList().get(0)\n\n assertEquals(12, result.getLong(9))\n assertEquals(\"John Henry Bonham\", result.getString(4))\n }\n\n @Test\n def `should insert index while insert nodes`(): Unit = {\n val ds = (1 to 10)\n .map(i => i.toString)\n .toDF(\"surname\")\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":Person:Customer\")\n .option(\"node.keys\", \"surname\")\n .option(\"schema.optimization.type\", \"INDEX\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:Person:Customer)\n |RETURN p.surname AS surname\n |\"\"\".stripMargin\n ).list().asScala\n .map(r => r.asMap().asScala)\n .toSet\n val expected = ds.collect().map(row => Map(\"surname\" -> row.getAs[String](\"surname\")))\n .toSet\n assertEquals(expected, records)\n\n val indexCount = SparkConnectorScalaSuiteIT.session().run(\n getIndexQueryCount\n )\n .single()\n .get(\"count\")\n .asLong()\n assertEquals(1, indexCount)\n\n SparkConnectorScalaSuiteIT.session().run(\"DROP INDEX spark_INDEX_Person_surname\")\n }\n\n private def getIndexQueryCount: String = {\n val (uniqueKey, uniqueCondition) =\n if (TestUtil.neo4jVersion(SparkConnectorScalaSuiteIT.session()) >= Versions.NEO4J_5) {\n (\"owningConstraint\", \"owningConstraint IS NULL\")\n } else {\n (\"uniqueness\", \"uniqueness = 'NONUNIQUE'\")\n }\n\n s\"\"\"SHOW INDEXES YIELD labelsOrTypes, properties, $uniqueKey\n |WHERE labelsOrTypes = ['Person'] AND properties = ['surname'] AND $uniqueCondition\n |RETURN count(*) AS count\n |\"\"\".stripMargin\n }\n\n private def getConstraintQueryCount: String = {\n val (uniqueKey, uniqueCondition) =\n if (TestUtil.neo4jVersion(SparkConnectorScalaSuiteIT.session()) >= Versions.NEO4J_5) {\n (\"owningConstraint\", \"owningConstraint IS NOT NULL\")\n } else {\n (\"uniqueness\", \"uniqueness = 'UNIQUE'\")\n }\n s\"\"\"SHOW INDEXES YIELD labelsOrTypes, properties, $uniqueKey\n |WHERE labelsOrTypes = ['Person'] AND properties = ['surname'] AND $uniqueCondition\n |RETURN count(*) AS count\n |\"\"\".stripMargin\n }\n\n @Test\n def `should create constraint when insert nodes`(): Unit = {\n val ds = (1 to 10)\n .map(i => i.toString)\n .toDF(\"surname\")\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":Person:Customer\")\n .option(\"node.keys\", \"surname\")\n .option(\"schema.optimization.type\", \"NODE_CONSTRAINTS\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:Person:Customer)\n |RETURN p.surname AS surname\n |\"\"\".stripMargin\n ).list().asScala\n .map(r => r.asMap().asScala)\n .toSet\n val expected = ds.collect().map(row => Map(\"surname\" -> row.getAs[String](\"surname\")))\n .toSet\n assertEquals(expected, records)\n\n val constraintCount = SparkConnectorScalaSuiteIT.session().run(\n getConstraintQueryCount\n )\n .single()\n .get(\"count\")\n .asLong()\n assertEquals(1, constraintCount)\n SparkConnectorScalaSuiteIT.session().run(\"DROP CONSTRAINT spark_NODE_CONSTRAINTS_Person_surname\")\n }\n\n @Test\n def `should not create constraint when insert nodes because they already exist`(): Unit = {\n SparkConnectorScalaSuiteIT.session().run(\n \"CREATE CONSTRAINT person_surname FOR (p:Person) REQUIRE (p.surname) IS UNIQUE\"\n )\n\n val ds = (1 to 10)\n .map(i => i.toString)\n .toDF(\"surname\")\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":Person:Customer\")\n .option(\"node.keys\", \"surname\")\n .option(\"schema.optimization.type\", \"NODE_CONSTRAINTS\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:Person:Customer)\n |RETURN p.surname AS surname\n |\"\"\".stripMargin\n ).list().asScala\n .map(r => r.asMap().asScala)\n .toSet\n val expected = ds.collect().map(row => Map(\"surname\" -> row.getAs[String](\"surname\")))\n .toSet\n assertEquals(expected, records)\n\n val constraintCount = SparkConnectorScalaSuiteIT.session().run(\n getConstraintQueryCount\n )\n .single()\n .get(\"count\")\n .asLong()\n assertEquals(1, constraintCount)\n SparkConnectorScalaSuiteIT.session().run(\"DROP CONSTRAINT person_surname\")\n }\n\n @Test\n def `should insert indexes while insert with query`(): Unit = {\n val ds = (1 to 10)\n .map(i => i.toString)\n .toDF(\"surname\")\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Overwrite)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":Person:Customer\")\n .option(\"node.keys\", \"surname\")\n .option(\"schema.optimization.type\", \"INDEX\")\n .save()\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"query\", \"CREATE (n:MyNode{fullName: event.name + event.surname, age: event.age - 10})\")\n .option(\"batch.size\", \"11\")\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:Person:Customer)\n |RETURN p.surname AS surname\n |\"\"\".stripMargin\n ).list().asScala\n .map(r => r.asMap().asScala)\n .toSet\n val expected = ds.collect().map(row => Map(\"surname\" -> row.getAs[String](\"surname\")))\n .toSet\n assertEquals(expected, records)\n\n val indexCount = SparkConnectorScalaSuiteIT.session()\n .run(getIndexQueryCount)\n .single()\n .get(\"count\")\n .asLong()\n assertEquals(1, indexCount)\n\n SparkConnectorScalaSuiteIT.session().run(\"DROP INDEX spark_INDEX_Person_surname\")\n }\n\n @Test\n def `should manage script passing the data to the executors`(): Unit = {\n val ds = Seq(SimplePerson(\"Andrea\", \"Santurbano\"), SimplePerson(\"Davide\", \"Fantuzzi\")).toDS()\n .repartition(2)\n\n ds.write\n .format(classOf[DataSource].getName)\n .mode(SaveMode.Append)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"CREATE (n:Person{fullName: event.name + ' ' + event.surname, age: scriptResult[0].age[event.name]})\"\n )\n .option(\n \"script\",\n \"\"\"CREATE INDEX person_surname FOR (p:Person) ON (p.surname);\n |CREATE CONSTRAINT product_name_sku FOR (p:Product)\n | REQUIRE (p.name, p.sku)\n | IS NODE KEY;\n |RETURN {Andrea: 36, Davide: 32} AS age;\n |\"\"\".stripMargin\n )\n .save()\n\n val records = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH (p:Person)\n |WHERE (p.fullName = 'Andrea Santurbano' AND p.age = 36)\n |OR (p.fullName = 'Davide Fantuzzi' AND p.age = 32)\n |RETURN count(p) AS count\n |\"\"\".stripMargin\n )\n .single()\n .get(\"count\")\n .asLong()\n val expected = ds.count\n assertEquals(expected, records)\n\n val uniqueFieldName = if (TestUtil.neo4jVersion(SparkConnectorScalaSuiteIT.session()) >= Versions.NEO4J_5)\n \"owningConstraint\"\n else \"uniqueness\"\n val (indexCondition, uniqueCondition) =\n if (TestUtil.neo4jVersion(SparkConnectorScalaSuiteIT.session()) >= Versions.NEO4J_5) {\n (s\"$uniqueFieldName IS NULL\", s\"$uniqueFieldName IS NOT NULL\")\n } else {\n (s\"$uniqueFieldName = 'NONUNIQUE'\", s\"$uniqueFieldName = 'UNIQUE'\")\n }\n val query =\n s\"\"\"SHOW INDEXES YIELD labelsOrTypes, properties, $uniqueFieldName\n |WHERE (labelsOrTypes = ['Person'] AND properties = ['surname'] AND $indexCondition)\n |OR (labelsOrTypes = ['Product'] AND properties = ['name', 'sku'] AND $uniqueCondition)\n |RETURN count(*) AS count\n |\"\"\".stripMargin\n val constraintCount = SparkConnectorScalaSuiteIT.session()\n .run(query)\n .single()\n .get(\"count\")\n .asLong()\n assertEquals(2, constraintCount)\n SparkConnectorScalaSuiteIT.session().run(\"DROP INDEX person_surname\")\n SparkConnectorScalaSuiteIT.session().run(\"DROP CONSTRAINT product_name_sku\")\n }\n\n @Test\n def `should work create source node and match target node`() {\n val data = Seq(\n (12, \"John Bonham\", \"Drums\"),\n (19, \"John Mayer\", \"Guitar\"),\n (32, \"John Scofield\", \"Guitar\"),\n (15, \"John Butler\", \"Guitar\")\n )\n SparkConnectorScalaSuiteIT.session().run(\"CREATE \" + data\n .map(_._3)\n .toSet[String]\n .map(instrument => s\"(:Instrument{name: '$instrument'})\")\n .mkString(\", \"))\n val musicDf = data.toDF(\"experience\", \"name\", \"instrument\")\n\n musicDf.write\n .mode(SaveMode.Overwrite)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"name\")\n .option(\"relationship.target.save.mode\", \"match\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .save\n\n val count = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH p = (:Musician)-[:PLAYS]->(:Instrument)\n |RETURN count(p) AS count\"\"\".stripMargin\n )\n .single()\n .get(\"count\")\n .asLong()\n\n assertEquals(data.size, count)\n }\n\n @Test\n def `should work match source node and merge target node`() {\n SparkConnectorScalaSuiteIT.session().run(\n \"CREATE CONSTRAINT musician_name FOR (m:Musician) REQUIRE (m.name) IS UNIQUE\"\n )\n val data = Seq(\n (12, \"John Bonham\", \"Drums\"),\n (19, \"John Mayer\", \"Guitar\"),\n (32, \"John Scofield\", \"Guitar\"),\n (15, \"John Butler\", \"Guitar\")\n )\n SparkConnectorScalaSuiteIT.session().run(\"CREATE \" + data\n .map(_._2)\n .toSet[String]\n .map(name => s\"(:Musician{name: '$name'})\")\n .mkString(\", \"))\n val musicDf = data.toDF(\"experience\", \"name\", \"instrument\")\n\n musicDf.repartition(1).write\n .mode(SaveMode.Overwrite)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"match\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"name\")\n .option(\"relationship.target.save.mode\", \"overwrite\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .save\n\n val count = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH p = (:Musician)-[:PLAYS]->(:Instrument)\n |RETURN count(p) AS count\"\"\".stripMargin\n )\n .single()\n .get(\"count\")\n .asLong()\n\n assertEquals(data.size, count)\n\n SparkConnectorScalaSuiteIT.session().run(\"DROP CONSTRAINT musician_name\")\n }\n\n @Test\n def `should work match source node and merge target node with odd chars`() {\n val data = Seq(\n (12, \"John Bonham\", \"Drums\"),\n (19, \"John Mayer\", \"Guitar\"),\n (32, \"John Scofield\", \"Guitar\"),\n (15, \"John Butler\", \"Guitar\")\n )\n val musicDf = data.toDF(\"experience\", \"who:name\", \"instrument\")\n\n musicDf.repartition(1).write\n .mode(SaveMode.Overwrite)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"overwrite\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"`who:name`\")\n .option(\"relationship.target.save.mode\", \"overwrite\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .save\n\n val count = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"MATCH p = (:Musician)-[:PLAYS]->(:Instrument)\n |RETURN count(p) AS count\"\"\".stripMargin\n )\n .single()\n .get(\"count\")\n .asLong()\n\n assertEquals(data.size, count)\n }\n\n @Test\n def shouldWriteComplexDF(): Unit = {\n val data = Seq(\n (\n \"Cuba Gooding Jr.\",\n 1,\n \"2022-06-07 00:00:00\",\n Seq(Map(\"product_id\" -> 1, \"quantity\" -> 2), Map(\"product_id\" -> 2, \"quantity\" -> 4))\n ),\n (\n \"Tom Hanks\",\n 2,\n \"2022-07-07 00:00:00\",\n Seq(Map(\"product_id\" -> 11, \"quantity\" -> 2), Map(\"product_id\" -> 22, \"quantity\" -> 4))\n )\n ).toDF(\"actor_name\", \"order_id\", \"order_date\", \"products\")\n data.write\n .mode(SaveMode.Overwrite)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\n \"query\",\n \"\"\"\n |MERGE (person:Person {name: event.actor_name})\n |CREATE (order:Order {id: event.order_id, date: datetime(replace(event.order_date, ' ', 'T'))})\n |MERGE (person)-[:CREATED]->(order)\n |WITH event, person, order\n |UNWIND event.products AS product_order\n |MERGE (product:Product {id: product_order.product_id})\n |CREATE (order)-[:CONTAINS{quantityOrdered: product_order.quantity}]->(product)\n |\"\"\".stripMargin\n )\n .save()\n\n val actual = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (p:Person)-[cr:CREATED]->(o:Order)-[co:CONTAINS]->(pr:Product)\n |WITH p, pr, o, co\n |ORDER BY p.name, pr.id\n |RETURN p.name AS name, o.id AS order, collect({id: pr.id, quantity: co.quantityOrdered}) AS products\n |\"\"\".stripMargin\n )\n .list()\n .asScala\n .map(_.asMap())\n .toSet\n .asJava\n val expected = Set(\n Map(\n \"name\" -> \"Cuba Gooding Jr.\",\n \"order\" -> 1L,\n \"products\" -> List(\n Map(\"id\" -> 1L, \"quantity\" -> 2L).asJava,\n Map(\"id\" -> 2L, \"quantity\" -> 4L).asJava\n ).asJava\n ).asJava,\n Map(\n \"name\" -> \"Tom Hanks\",\n \"order\" -> 2L,\n \"products\" -> List(\n Map(\"id\" -> 11L, \"quantity\" -> 2L).asJava,\n Map(\"id\" -> 22L, \"quantity\" -> 4L).asJava\n ).asJava\n ).asJava\n ).asJava\n assertEquals(expected, actual)\n }\n\n @Test\n def shouldFix502(): Unit = {\n val data = Seq(\n (\"Foo\", 1, Map(\"inner\" -> Map(\"key\" -> \"innerValue\"))),\n (\"Bar\", 1, Map(\"inner\" -> Map(\"key\" -> \"innerValue1\")))\n ).toDF(\"id\", \"time\", \"table\")\n data.write\n .mode(SaveMode.Append)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":MyNodeWithMapFlattend\")\n .save()\n val count: Long = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (n:MyNodeWithMapFlattend)\n |WHERE (\n | properties(n) = {id: 'Foo', time: 1, `table.inner.key`: 'innerValue'}\n | OR properties(n) = {id: 'Bar', time: 1, `table.inner.key`: 'innerValue1'}\n |)\n |RETURN count(n)\n |\"\"\".stripMargin\n )\n .single()\n .get(0)\n .asLong()\n junit.Assert.assertEquals(2L, count)\n }\n\n @Test\n def shouldFix502WithCollisions(): Unit = {\n val data = Seq(\n (\"Foo\", 1, ListMap(\"key.inner\" -> Map(\"key\" -> \"innerValue\"), \"key\" -> Map(\"inner.key\" -> \"value\"))),\n (\"Bar\", 1, ListMap(\"key.inner\" -> Map(\"key\" -> \"innerValue1\"), \"key\" -> Map(\"inner.key\" -> \"value1\")))\n ).toDF(\"id\", \"time\", \"table\")\n data.write\n .mode(SaveMode.Append)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":MyNodeWithMapFlattend\")\n .save()\n val count: Long = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (n:MyNodeWithMapFlattend)\n |WHERE (\n | properties(n) = {id: 'Foo', time: 1, `table.key.inner.key`: 'value'}\n | OR properties(n) = {id: 'Bar', time: 1, `table.key.inner.key`: 'value1'}\n |)\n |RETURN count(n)\n |\"\"\".stripMargin\n )\n .single()\n .get(0)\n .asLong()\n junit.Assert.assertEquals(2L, count)\n }\n\n @Test\n def shouldFix502WithCollisionsAndAggregateValues(): Unit = {\n val data = Seq(\n (\"Foo\", 1, ListMap(\"key.inner\" -> Map(\"key\" -> \"innerValue\"), \"key\" -> Map(\"inner.key\" -> \"value\"))),\n (\"Bar\", 1, ListMap(\"key.inner\" -> Map(\"key\" -> \"innerValue1\"), \"key\" -> Map(\"inner.key\" -> \"value1\")))\n ).toDF(\"id\", \"time\", \"table\")\n data.write\n .mode(SaveMode.Append)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"labels\", \":MyNodeWithMapFlattend\")\n .option(\"schema.map.group.duplicate.keys\", true)\n .save()\n val count: Long = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (n:MyNodeWithMapFlattend)\n |WHERE (\n | properties(n) = {id: 'Foo', time: 1, `table.key.inner.key`: ['innerValue', 'value']}\n | OR properties(n) = {id: 'Bar', time: 1, `table.key.inner.key`: ['innerValue1', 'value1']}\n |)\n |RETURN count(n)\n |\"\"\".stripMargin\n )\n .single()\n .get(0)\n .asLong()\n junit.Assert.assertEquals(2L, count)\n }\n\n @Test\n def doesNotWriteNodePropertiesToRelationship(): Unit = {\n val data = Seq(\n (\"john\", \"The Matrix\", \"today\"),\n (\"jane\", \"Oppenheimer\", \"yesterday\"),\n (\"şaban\", \"Hababam Sınıfı\", \"two days ago\")\n ).toDF(\"username\", \"movie_title\", \"watch_time\")\n data.write\n .mode(SaveMode.Append)\n .format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n .option(\"relationship\", \"WATCHED\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.save.mode\", \"Overwrite\")\n .option(\"relationship.source.labels\", \":User\")\n .option(\"relationship.source.node.keys\", \"username:name\")\n .option(\"relationship.target.save.mode\", \"Overwrite\")\n .option(\"relationship.target.labels\", \":Movie\")\n .option(\"relationship.target.node.keys\", \"movie_title:title\")\n .save()\n val rows = SparkConnectorScalaSuiteIT.session().run(\n \"\"\"\n |MATCH (:User)-[r:WATCHED]->(:Movie)\n |WITH r\n |ORDER BY r.watch_time ASC\n |RETURN collect(r{.*})\n |\"\"\".stripMargin\n )\n .single()\n .get(0)\n .asList((value: Value) => value.asMap().asScala)\n .asScala\n junit.Assert.assertEquals(\n List(\n Map(\"watch_time\" -> \"today\"),\n Map(\"watch_time\" -> \"two days ago\"),\n Map(\"watch_time\" -> \"yesterday\")\n ),\n rows\n )\n }\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/DefaultConfigTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.junit.Assert.assertEquals\nimport org.junit.Test\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.summary.ResultSummary\n\nclass DefaultConfigTSE extends SparkConnectorScalaBaseTSE {\n\n @Test\n def `when session has default parameters it should use those instead of requiring options`(): Unit = {\n\n SparkConnectorScalaSuiteIT.session()\n .writeTransaction(\n new TransactionWork[ResultSummary] {\n override def execute(tx: Transaction): ResultSummary = tx.run(\"CREATE (p:Person {name: 'Foobar'})\").consume()\n }\n )\n\n ss.conf.set(\"neo4j.url\", SparkConnectorScalaSuiteIT.server.getBoltUrl)\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"labels\", \"Person\")\n .load()\n\n assertEquals(df.count(), 1)\n }\n\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/GraphDataScienceIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.sql.types.ArrayType\nimport org.apache.spark.sql.types.DoubleType\nimport org.apache.spark.sql.types.LongType\nimport org.apache.spark.sql.types.MapType\nimport org.apache.spark.sql.types.StringType\nimport org.apache.spark.sql.types.StructField\nimport org.apache.spark.sql.types.StructType\nimport org.junit.After\nimport org.junit.Assert.assertEquals\nimport org.junit.Assert.assertTrue\nimport org.junit.Assert.fail\nimport org.junit.Assume\nimport org.junit.Test\nimport org.neo4j.Closeables.use\nimport org.neo4j.driver.Transaction\n\nimport scala.math.Ordering.Implicits.infixOrderingOps\n\nclass GraphDataScienceIT extends SparkConnectorScalaSuiteWithGdsBase {\n\n @After\n def cleanData(): Unit = {\n use(SparkConnectorScalaSuiteWithGdsBase.session(\"system\")) { session =>\n session.run(\"CREATE OR REPLACE DATABASE neo4j WAIT 30 seconds\").consume()\n }\n\n use(SparkConnectorScalaSuiteWithGdsBase.session()) { session =>\n session\n .writeTransaction((tx: Transaction) => {\n tx.run(\n \"\"\"\n |CALL gds.graph.list() YIELD graphName\n |WITH graphName AS g\n |CALL gds.graph.drop(g) YIELD graphName\n |RETURN *\n |\"\"\".stripMargin\n ).consume()\n })\n }\n }\n\n @Test\n def shouldReturnThePageRank(): Unit = {\n initForPageRank()\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"gds\", \"gds.pageRank.stream\")\n .option(\"gds.graphName\", \"myGraph\")\n .option(\"gds.configuration.concurrency\", \"2\")\n .load()\n assertEquals(df.count(), 8)\n df.show(false)\n\n assertEquals(StructType(Array(StructField(\"nodeId\", LongType), StructField(\"score\", DoubleType))), df.schema)\n\n val dfEstimate = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"gds\", \"gds.pageRank.stream.estimate\")\n .option(\"gds.graphNameOrConfiguration\", \"myGraph\")\n .option(\"gds.algoConfiguration.concurrency\", \"2\")\n .load()\n assertEquals(dfEstimate.count(), 1)\n dfEstimate.show(false)\n\n assertEquals(\n StructType(\n Array(\n StructField(\"requiredMemory\", StringType),\n StructField(\"treeView\", StringType),\n StructField(\"mapView\", MapType(StringType, StringType)),\n StructField(\"bytesMin\", LongType),\n StructField(\"bytesMax\", LongType),\n StructField(\"nodeCount\", LongType),\n StructField(\"relationshipCount\", LongType),\n StructField(\"heapPercentageMin\", DoubleType),\n StructField(\"heapPercentageMax\", DoubleType)\n )\n ),\n dfEstimate.schema\n )\n }\n\n @Test\n def shouldFailWithUnsupportedOptions(): Unit = {\n initForPageRank()\n\n def run(options: Map[String, String], error: String): Unit = {\n try {\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .options(options)\n .load()\n .show(false)\n fail(\"Expected to throw an exception\")\n } catch {\n case iae: IllegalArgumentException =>\n assertTrue(iae.getMessage.equals(error))\n case _: Throwable =>\n fail(s\"should be thrown a ${classOf[IllegalArgumentException].getName}\")\n }\n }\n\n run(\n Map(\n \"gds\" -> \"gds.pageRank.stream\",\n \"gds.graphName\" -> \"myGraph\",\n \"gds.configuration.concurrency\" -> \"2\",\n \"partitions\" -> \"2\"\n ),\n \"For GDS queries we support only one partition\"\n )\n\n run(\n Map(\n \"gds\" -> \"gds.pageRank.write\",\n \"gds.graphName\" -> \"myGraph\",\n \"gds.configuration.concurrency\" -> \"2\"\n ),\n \"You cannot execute GDS mutate or write procedure in a read query\"\n )\n\n run(\n Map(\n \"gds\" -> \"gds.pageRank.mutate\",\n \"gds.graphName\" -> \"myGraph\",\n \"gds.configuration.concurrency\" -> \"2\"\n ),\n \"You cannot execute GDS mutate or write procedure in a read query\"\n )\n }\n\n @Test\n def shouldWorkWithMapReturn(): Unit = {\n initForHits()\n\n val procName = if (TestUtil.gdsVersion(SparkConnectorScalaSuiteWithGdsBase.session()) >= Versions.GDS_2_5)\n \"gds.hits.stream\"\n else \"gds.alpha.hits.stream\"\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"gds\", procName)\n .option(\"gds.graphName\", \"myGraph\")\n .option(\"gds.configuration.hitsIterations\", \"20\")\n .load()\n assertEquals(df.count(), 9)\n df.show(false)\n\n assertEquals(\n StructType(Array(StructField(\"nodeId\", LongType), StructField(\"values\", MapType(StringType, StringType)))),\n df.schema\n )\n }\n\n @Test\n def shouldWorkWithPathReturn(): Unit = {\n initForYens()\n\n val sourceTargetNodes = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"labels\", \"Location\")\n .load()\n .where(\"name IN ('A', 'F')\")\n .orderBy(\"name\")\n .collect()\n\n val (sourceId, targetId) = (sourceTargetNodes(0).getAs[Long](\"\"), sourceTargetNodes(1).getAs[Long](\"\"))\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"gds\", \"gds.shortestPath.yens.stream\")\n .option(\"gds.graphName\", \"myGraph\")\n .option(\"gds.configuration.sourceNode\", sourceId)\n .option(\"gds.configuration.targetNode\", targetId)\n .option(\"gds.configuration.k\", 3)\n .option(\"gds.configuration.relationshipWeightProperty\", \"cost\")\n .load()\n assertEquals(df.count(), 3)\n df.show(false)\n\n assertEquals(\n StructType(\n Array(\n StructField(\"index\", LongType),\n StructField(\"sourceNode\", LongType),\n StructField(\"targetNode\", LongType),\n StructField(\"totalCost\", DoubleType),\n StructField(\"nodeIds\", ArrayType(LongType)),\n StructField(\"costs\", ArrayType(DoubleType)),\n StructField(\"path\", StringType)\n )\n ),\n df.schema\n )\n\n val (graphNameParam, algoConfigurationParam) =\n if (TestUtil.gdsVersion(SparkConnectorScalaSuiteWithGdsBase.session()) >= Versions.GDS_2_4)\n (\"graphName\", \"configuration\")\n else (\"graphNameOrConfiguration\", \"algoConfiguration\")\n val dfEstimate = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"gds\", \"gds.shortestPath.yens.stream.estimate\")\n .option(s\"gds.$graphNameParam\", \"myGraph\")\n .option(s\"gds.$algoConfigurationParam.sourceNode\", sourceId)\n .option(s\"gds.$algoConfigurationParam.targetNode\", targetId)\n .option(s\"gds.$algoConfigurationParam.k\", 3)\n .option(s\"gds.$algoConfigurationParam.relationshipWeightProperty\", \"cost\")\n .load()\n assertEquals(dfEstimate.count(), 1)\n dfEstimate.show(false)\n\n assertEquals(\n StructType(\n Array(\n StructField(\"requiredMemory\", StringType),\n StructField(\"treeView\", StringType),\n StructField(\"mapView\", MapType(StringType, StringType)),\n StructField(\"bytesMin\", LongType),\n StructField(\"bytesMax\", LongType),\n StructField(\"nodeCount\", LongType),\n StructField(\"relationshipCount\", LongType),\n StructField(\"heapPercentageMin\", DoubleType),\n StructField(\"heapPercentageMax\", DoubleType)\n )\n ),\n dfEstimate.schema\n )\n }\n\n private def initForYens(): Unit = {\n SparkConnectorScalaSuiteWithGdsBase.session()\n .writeTransaction((tx: Transaction) => {\n tx.run(\n \"\"\"\n |CREATE (a:Location {name: 'A'}),\n | (b:Location {name: 'B'}),\n | (c:Location {name: 'C'}),\n | (d:Location {name: 'D'}),\n | (e:Location {name: 'E'}),\n | (f:Location {name: 'F'}),\n | (a)-[:ROAD {cost: 50}]->(b),\n | (a)-[:ROAD {cost: 50}]->(c),\n | (a)-[:ROAD {cost: 100}]->(d),\n | (b)-[:ROAD {cost: 40}]->(d),\n | (c)-[:ROAD {cost: 40}]->(d),\n | (c)-[:ROAD {cost: 80}]->(e),\n | (d)-[:ROAD {cost: 30}]->(e),\n | (d)-[:ROAD {cost: 80}]->(f),\n | (e)-[:ROAD {cost: 40}]->(f);\n |\"\"\".stripMargin\n ).consume()\n })\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"gds\", \"gds.graph.project\")\n .option(\"gds.graphName\", \"myGraph\")\n .option(\"gds.nodeProjection\", \"Location\")\n .option(\"gds.relationshipProjection\", \"ROAD\")\n .option(\"gds.configuration.relationshipProperties\", \"cost\")\n .load()\n .show(false)\n }\n\n @Test\n def shouldWorkWithKNearest(): Unit = {\n SparkConnectorScalaSuiteWithGdsBase.session()\n .writeTransaction((tx: Transaction) => {\n tx.run(\n \"\"\"\n |CREATE (alice:Person {name: 'Alice', age: 24, lotteryNumbers: [1, 3], embedding: [1.0, 3.0]})\n |CREATE (bob:Person {name: 'Bob', age: 73, lotteryNumbers: [1, 2, 3], embedding: [2.1, 1.6]})\n |CREATE (carol:Person {name: 'Carol', age: 24, lotteryNumbers: [3], embedding: [1.5, 3.1]})\n |CREATE (dave:Person {name: 'Dave', age: 48, lotteryNumbers: [2, 4], embedding: [0.6, 0.2]})\n |CREATE (eve:Person {name: 'Eve', age: 67, lotteryNumbers: [1, 5], embedding: [1.8, 2.7]});\n |\"\"\".stripMargin\n ).consume()\n })\n\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"gds\", \"gds.graph.project\")\n .option(\"gds.graphName\", \"myGraph\")\n .option(\"gds.nodeProjection.Person.properties\", \"['age','lotteryNumbers','embedding']\")\n .option(\"gds.relationshipProjection\", \"*\")\n .load()\n .show(false)\n\n val df = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"gds\", \"gds.knn.stream\")\n .option(\"gds.graphName\", \"myGraph\")\n .option(\"gds.configuration.topK\", 1)\n .option(\"gds.configuration.nodeProperties\", \"['age']\")\n .option(\"gds.configuration.randomSeed\", 1337)\n .option(\"gds.configuration.concurrency\", 1)\n .option(\"gds.configuration.sampleRate\", 1.0)\n .option(\"gds.configuration.deltaThreshold\", 0.0)\n .load()\n\n assertEquals(df.count(), 5)\n df.show(false)\n\n assertEquals(\n StructType(\n Array(\n StructField(\"node1\", LongType),\n StructField(\"node2\", LongType),\n StructField(\"similarity\", DoubleType)\n )\n ),\n df.schema\n )\n\n val dfEstimate = ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"gds\", \"gds.knn.stream.estimate\")\n .option(\"gds.graphNameOrConfiguration\", \"myGraph\")\n .option(\"gds.algoConfiguration.topK\", 1)\n .option(\"gds.algoConfiguration.nodeProperties\", \"['age']\")\n .option(\"gds.algoConfiguration.randomSeed\", 1337)\n .option(\"gds.algoConfiguration.concurrency\", 1)\n .option(\"gds.algoConfiguration.sampleRate\", 1.0)\n .option(\"gds.algoConfiguration.deltaThreshold\", 0.0)\n .load()\n assertEquals(dfEstimate.count(), 1)\n dfEstimate.show(false)\n\n assertEquals(\n StructType(\n Array(\n StructField(\"requiredMemory\", StringType),\n StructField(\"treeView\", StringType),\n StructField(\"mapView\", MapType(StringType, StringType)),\n StructField(\"bytesMin\", LongType),\n StructField(\"bytesMax\", LongType),\n StructField(\"nodeCount\", LongType),\n StructField(\"relationshipCount\", LongType),\n StructField(\"heapPercentageMin\", DoubleType),\n StructField(\"heapPercentageMax\", DoubleType)\n )\n ),\n dfEstimate.schema\n )\n }\n\n private def initForPageRank(): Unit = {\n SparkConnectorScalaSuiteWithGdsBase.session()\n .writeTransaction((tx: Transaction) => {\n tx.run(\n \"\"\"\n |CREATE\n | (home:Page {name:'Home'}),\n | (about:Page {name:'About'}),\n | (product:Page {name:'Product'}),\n | (links:Page {name:'Links'}),\n | (a:Page {name:'Site A'}),\n | (b:Page {name:'Site B'}),\n | (c:Page {name:'Site C'}),\n | (d:Page {name:'Site D'}),\n |\n | (home)-[:LINKS {weight: 0.2}]->(about),\n | (home)-[:LINKS {weight: 0.2}]->(links),\n | (home)-[:LINKS {weight: 0.6}]->(product),\n | (about)-[:LINKS {weight: 1.0}]->(home),\n | (product)-[:LINKS {weight: 1.0}]->(home),\n | (a)-[:LINKS {weight: 1.0}]->(home),\n | (b)-[:LINKS {weight: 1.0}]->(home),\n | (c)-[:LINKS {weight: 1.0}]->(home),\n | (d)-[:LINKS {weight: 1.0}]->(home),\n | (links)-[:LINKS {weight: 0.8}]->(home),\n | (links)-[:LINKS {weight: 0.05}]->(a),\n | (links)-[:LINKS {weight: 0.05}]->(b),\n | (links)-[:LINKS {weight: 0.05}]->(c),\n | (links)-[:LINKS {weight: 0.05}]->(d);\n |\"\"\".stripMargin\n ).consume()\n })\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"gds\", \"gds.graph.project\")\n .option(\"gds.graphName\", \"myGraph\")\n .option(\"gds.nodeProjection\", \"Page\")\n .option(\"gds.relationshipProjection\", \"LINKS\")\n .option(\"gds.configuration.relationshipProperties\", \"weight\")\n .load()\n .show(false)\n }\n\n private def initForHits(): Unit = {\n Assume.assumeTrue(TestUtil.neo4jVersion(SparkConnectorScalaSuiteWithGdsBase.session()) >= Versions.NEO4J_5)\n SparkConnectorScalaSuiteWithGdsBase.session()\n .writeTransaction((tx: Transaction) => {\n tx.run(\n \"\"\"\n CREATE\n | (a:Website {name: 'A'}),\n | (b:Website {name: 'B'}),\n | (c:Website {name: 'C'}),\n | (d:Website {name: 'D'}),\n | (e:Website {name: 'E'}),\n | (f:Website {name: 'F'}),\n | (g:Website {name: 'G'}),\n | (h:Website {name: 'H'}),\n | (i:Website {name: 'I'}),\n |\n | (a)-[:LINK]->(b),\n | (a)-[:LINK]->(c),\n | (a)-[:LINK]->(d),\n | (b)-[:LINK]->(c),\n | (b)-[:LINK]->(d),\n | (c)-[:LINK]->(d),\n |\n | (e)-[:LINK]->(b),\n | (e)-[:LINK]->(d),\n | (e)-[:LINK]->(f),\n | (e)-[:LINK]->(h),\n |\n | (f)-[:LINK]->(g),\n | (f)-[:LINK]->(i),\n | (f)-[:LINK]->(h),\n | (g)-[:LINK]->(h),\n | (g)-[:LINK]->(i),\n | (h)-[:LINK]->(i);\n |\"\"\".stripMargin\n ).consume()\n })\n ss.read.format(classOf[DataSource].getName)\n .option(\"url\", SparkConnectorScalaSuiteWithGdsBase.server.getBoltUrl)\n .option(\"gds\", \"gds.graph.project\")\n .option(\"gds.graphName\", \"myGraph\")\n .option(\"gds.nodeProjection\", \"Website\")\n .option(\"gds.relationshipProjection.LINK.indexInverse\", \"true\")\n .load()\n .show(false)\n }\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/ReauthenticationIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.junit.AfterClass\nimport org.junit.Assert.assertEquals\nimport org.junit.BeforeClass\nimport org.junit.Test\nimport org.neo4j.Neo4jContainerExtension\nimport org.neo4j.driver.AuthTokens\nimport org.neo4j.driver.Driver\nimport org.neo4j.driver.GraphDatabase\nimport org.neo4j.spark.ReauthenticationIT.KEYCLOAK\nimport org.neo4j.spark.ReauthenticationIT.NEO4J\nimport org.neo4j.spark.SparkConnectorScalaSuiteIT.ss\nimport org.slf4j.LoggerFactory\nimport org.testcontainers.containers.GenericContainer\nimport org.testcontainers.containers.Network\nimport org.testcontainers.containers.output.Slf4jLogConsumer\nimport org.testcontainers.containers.wait.strategy.Wait\nimport org.testcontainers.containers.wait.strategy.WaitAllStrategy\nimport org.testcontainers.utility.MountableFile\n\nimport java.time.Duration.ofMinutes\n\nobject ReauthenticationIT {\n\n private val log = LoggerFactory.getLogger(classOf[ReauthenticationIT])\n\n private class TestKeycloakContainer(image: String) extends GenericContainer[TestKeycloakContainer](image) {\n def getHttpPort: Integer = this.getMappedPort(8080)\n }\n\n private val NETWORK = Network.newNetwork\n\n private val KEYCLOAK = new TestKeycloakContainer(\"quay.io/keycloak/keycloak:26.2.5\")\n .withNetwork(NETWORK)\n .withEnv(\"KC_BOOTSTRAP_ADMIN_USERNAME\", \"admin\")\n .withEnv(\"KC_BOOTSTRAP_ADMIN_PASSWORD\", \"admin\")\n .withNetworkAliases(\"keycloak\")\n .withExposedPorts(8080, 9000, 8443)\n .withCopyFileToContainer(\n MountableFile.forClasspathResource(\"/neo4j-keycloak.jks\"),\n \"/opt/keycloak/conf/server.keystore\"\n )\n .withEnv(\"KC_HTTPS_KEY_STORE_FILE\", \"/opt/keycloak/conf/server.keystore\")\n .withEnv(\"KC_HTTPS_KEY_STORE_PASSWORD\", \"testpwd\")\n .withEnv(\"KC_HTTPS_KEY_PASSWORD\", \"testpwd\")\n .withEnv(\"KC_HEALTH_ENABLED\", \"true\")\n .withCopyFileToContainer(\n MountableFile.forClasspathResource(\"/neo4j-sso-test-realm.json\"),\n \"/opt/keycloak/data/import/neo4j-sso-test-realm.json\"\n )\n .withEnv(\"KC_HOSTNAME\", \"https://keycloak:8443\")\n .withEnv(\"KC_HOSTNAME_BACKCHANNEL_DYNAMIC\", \"true\")\n .waitingFor(\n new WaitAllStrategy(WaitAllStrategy.Mode.WITH_INDIVIDUAL_TIMEOUTS_ONLY)\n .withStrategy(Wait.forListeningPort().withStartupTimeout(ofMinutes(2)))\n .withStrategy(\n Wait.forHttp(\"/health/started\")\n .forPort(9000)\n .usingTls()\n .allowInsecure()\n .forStatusCode(200)\n .withStartupTimeout(java.time.Duration.ofMinutes(5))\n )\n )\n .withStartupAttempts(3)\n .withLogConsumer(new Slf4jLogConsumer(log))\n .withCommand(\"start-dev --import-realm\")\n\n private val NEO4J = new Neo4jContainerExtension()\n .withNetwork(NETWORK)\n .withEnv(\"NEO4J_ACCEPT_LICENSE_AGREEMENT\", \"yes\")\n .withCopyFileToContainer(MountableFile.forClasspathResource(\"/neo4j-keycloak.jks\"), \"/tmp/keycloak.jks\")\n .withEnv(\n \"_JAVA_OPTIONS\",\n \"-Djavax.net.ssl.keyStore=/tmp/keycloak.jks -Djavax.net.ssl.keyStorePassword=testpwd -Djavax.net.ssl.trustStore=/tmp/keycloak.jks -Djavax.net.ssl.trustStorePassword=testpwd\"\n )\n .withNeo4jConfig(\"dbms.security.authentication_providers\", \"oidc-keycloak,native\")\n .withNeo4jConfig(\"dbms.security.authorization_providers\", \"oidc-keycloak,native\")\n .withNeo4jConfig(\"dbms.security.oidc.keycloak.display_name\", \"Keycloak\")\n .withNeo4jConfig(\"dbms.security.oidc.keycloak.auth_flow\", \"pkce\")\n .withNeo4jConfig(\n \"dbms.security.oidc.keycloak.well_known_discovery_uri\",\n \"https://keycloak:8443/realms/neo4j-sso-test/.well-known/openid-configuration\"\n )\n .withNeo4jConfig(\n \"dbms.security.oidc.keycloak.params\",\n \"client_id=neo4j-commons-client;response_type=code;scope=openid email roles\"\n )\n .withNeo4jConfig(\"dbms.security.oidc.keycloak.audience\", \"account\")\n .withNeo4jConfig(\"dbms.security.oidc.keycloak.issuer\", \"https://keycloak:8443/realms/neo4j-sso-test\")\n .withNeo4jConfig(\"dbms.security.oidc.keycloak.client_id\", \"neo4j-commons-client\")\n .withNeo4jConfig(\"dbms.security.oidc.keycloak.claims.username\", \"preferred_username\")\n .withNeo4jConfig(\"dbms.security.oidc.keycloak.claims.groups\", \"groups\")\n .withNeo4jConfig(\"dbms.security.auth_cache_ttl\", \"1s\")\n\n @BeforeClass\n def setUp(): Unit = {\n KEYCLOAK.start()\n NEO4J.start()\n }\n\n @AfterClass\n def tearDown() = {\n TestUtil.closeSafely(NEO4J)\n TestUtil.closeSafely(KEYCLOAK)\n TestUtil.closeSafely(NETWORK)\n }\n}\n\nclass ReauthenticationIT extends SparkConnectorScalaSuiteIT {\n\n @Test\n def createAnInstanceOfReAuthDriver(): Unit = {\n val options = Map(\n \"url\" -> NEO4J.getBoltUrl,\n \"authentication.type\" -> \"keycloak\",\n \"authentication.keycloak.username\" -> \"john-tester\",\n \"authentication.keycloak.password\" -> \"testerpwd\",\n \"authentication.keycloak.authServerUrl\" ->\n s\"http://${KEYCLOAK.getHost}:${KEYCLOAK.getHttpPort}\",\n \"authentication.keycloak.realm\" -> \"neo4j-sso-test\",\n \"authentication.keycloak.clientId\" -> \"neo4j-commons-client\",\n \"authentication.keycloak.clientSecret\" -> \"QNrSpbh0mxhnlYlI21UcBaz3Htb734vi\"\n )\n\n var driver: Driver = null\n try {\n driver = GraphDatabase.driver(NEO4J.getBoltUrl, AuthTokens.basic(\"neo4j\", NEO4J.getAdminPassword))\n driver.session().run(\" CREATE (n:Test {field: 42}) CREATE (t:Test {field: 45})\").consume()\n } finally {\n driver.close()\n }\n\n val df = ss.read.format(classOf[DataSource].getName)\n .options(options)\n .option(\"query\", \"MATCH (t:Test {field: 42}) RETURN t.field\")\n .load()\n .toDF()\n assertEquals(42, df.first().getLong(0))\n\n Thread.sleep(4000)\n val df2 = ss.read.format(classOf[DataSource].getName)\n .options(options)\n .option(\"query\", \"MATCH (t:Test {field: 45}) RETURN t.field\")\n .load()\n .toDF()\n assertEquals(45, df2.first().getLong(0))\n }\n\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/SparkConnector30ScalaSuiteIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.junit.runner.RunWith\nimport org.junit.runners.Suite\n\n@RunWith(classOf[Suite])\n@Suite.SuiteClasses(Array(\n classOf[DataSourceReaderTSE],\n classOf[DataSourceReaderNeo4jTSE],\n classOf[DataSourceWriterNeo4jTSE],\n classOf[DataSourceWriterTSE],\n classOf[DataSourceSchemaWriterTSE],\n classOf[DefaultConfigTSE],\n classOf[DataSourceStreamingReaderTSE],\n classOf[DataSourceStreamingWriterTSE],\n classOf[DataSourceReaderAggregationTSE]\n))\nclass SparkConnector30ScalaSuiteIT extends SparkConnectorScalaSuiteIT {}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/SparkConnector30ScalaSuiteWithApocIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.junit.runner.RunWith\nimport org.junit.runners.Suite\n\n@RunWith(classOf[Suite])\n@Suite.SuiteClasses(Array(\n classOf[DataSourceReaderWithApocTSE],\n classOf[DataSourceReaderNeo4jWithApocTSE],\n classOf[DataSourceReaderAggregationTSE]\n))\nclass SparkConnector30ScalaSuiteWithApocIT extends SparkConnectorScalaSuiteWithApocIT {}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/SparkConnectorAuraTest.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.SparkConf\nimport org.apache.spark.sql.SparkSession\nimport org.junit.AfterClass\nimport org.junit.Assert._\nimport org.junit.Assume.assumeTrue\nimport org.junit.Before\nimport org.junit.BeforeClass\nimport org.junit.Test\nimport org.neo4j.Closeables.use\nimport org.neo4j.driver._\nimport org.neo4j.spark.SparkConnectorAuraTest._\n\nobject SparkConnectorAuraTest {\n private var neo4j: Driver = _\n private val username: Option[String] = Option[String](System.getenv(\"AURA_USER\"))\n private val password: Option[String] = Option[String](System.getenv(\"AURA_PASSWORD\"))\n private val url: Option[String] = Option[String](System.getenv(\"AURA_URI\"))\n\n var sparkSession: SparkSession = _\n\n @BeforeClass\n def setUpClass(): Unit = {\n assumeTrue(username.isDefined)\n assumeTrue(password.isDefined)\n assumeTrue(url.isDefined)\n\n sparkSession = SparkSession.builder()\n .config(new SparkConf()\n .setAppName(\"neoTest\")\n .setMaster(\"local[*]\")\n .set(\"spark.driver.host\", \"127.0.0.1\"))\n .getOrCreate()\n\n neo4j = GraphDatabase.driver(url.get, AuthTokens.basic(username.get, password.get))\n }\n\n @AfterClass\n def tearDown(): Unit = {\n TestUtil.closeSafely(neo4j)\n TestUtil.closeSafely(sparkSession)\n }\n}\n\nclass SparkConnectorAuraTest {\n\n val ss: SparkSession = SparkSession.builder().getOrCreate()\n\n import ss.implicits._\n\n @Before\n def setUp(): Unit = {\n use(neo4j.session(SessionConfig.forDatabase(\"system\"))) {\n session =>\n session.run(\"CREATE OR REPLACE DATABASE neo4j WAIT 30 seconds\").consume()\n }\n }\n\n @Test\n def shouldWriteToAndReadFromAura(): Unit = {\n val df = Seq((\"John Bonham\", \"Drums\", 12), (\"John Mayer\", \"Guitar\", 8))\n .toDF(\"name\", \"instrument\", \"experience\")\n\n df.write\n .mode(\"Overwrite\")\n .format(classOf[DataSource].getName)\n .option(\"url\", url.get)\n .option(\"authentication.type\", \"basic\")\n .option(\"authentication.basic.username\", username.get)\n .option(\"authentication.basic.password\", password.get)\n .option(\"relationship\", \"PLAYS\")\n .option(\"relationship.source.save.mode\", \"Append\")\n .option(\"relationship.target.save.mode\", \"Append\")\n .option(\"relationship.save.strategy\", \"keys\")\n .option(\"relationship.source.labels\", \":Musician\")\n .option(\"relationship.source.node.keys\", \"name:name\")\n .option(\"relationship.target.labels\", \":Instrument\")\n .option(\"relationship.target.node.keys\", \"instrument:name\")\n .save()\n\n val results = sparkSession.read.format(classOf[DataSource].getName)\n .option(\"url\", url.get)\n .option(\"authentication.type\", \"basic\")\n .option(\"authentication.basic.username\", username.get)\n .option(\"authentication.basic.password\", password.get)\n .option(\"labels\", \"Musician\")\n .load()\n .collectAsList()\n\n assertEquals(2, results.size())\n }\n}\n" }, { "path": "spark-3/src/test/scala/org/neo4j/spark/TransactionTimeoutIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.SparkException\nimport org.apache.spark.sql.SparkSession\nimport org.junit.AfterClass\nimport org.junit.Assert.assertEquals\nimport org.junit.Assert.assertThrows\nimport org.junit.Assert.assertTrue\nimport org.junit.BeforeClass\nimport org.junit.Test\nimport org.neo4j.Neo4jContainerExtension\nimport org.neo4j.driver.exceptions.ClientException\nimport org.neo4j.spark.SparkConnectorScalaSuiteWithApocIT.conf\nimport org.neo4j.spark.SparkConnectorScalaSuiteWithApocIT.server\nimport org.neo4j.spark.SparkConnectorScalaSuiteWithApocIT.ss\nimport org.neo4j.spark.TransactionTimeoutIT.NEO4J_LOW_TX_TIMEOUT\n\nimport java.util.TimeZone\n\nclass TransactionTimeoutIT extends SparkConnectorScalaSuiteWithApocIT {\n\n @Test\n def sparkConnectorRespectsTransactionTimeout(): Unit = {\n val cypher = \"UNWIND range(1, 3) AS i \" +\n \"CALL apoc.util.sleep(1000) \" +\n \"RETURN i as number\"\n val df = ss.read.format(\"org.neo4j.spark.DataSource\")\n .option(\"url\", server.getBoltUrl)\n .option(\"authentication.basic.username\", \"neo4j\")\n .option(\"authentication.basic.password\", server.getAdminPassword)\n .option(\"db.transaction.timeout\", \"4000\")\n .option(\"query\", cypher)\n .load()\n .toDF()\n\n val results = df.select(\"number\").rdd.map(_.getLong(0)).collect().toList\n val expected = Range.inclusive(1, 3).map(_.toLong).toList\n\n assertEquals(expected, results)\n }\n\n @Test\n def sparkConnectorFailsWithTransactionTimeoutWhenSetOnSessionLevel(): Unit = {\n val newConf = conf.clone().set(\"neo4j.url\", server.getBoltUrl)\n .set(\"neo4j.authentication.basic.username\", \"neo4j\")\n .set(\"neo4j.authentication.basic.password\", server.getAdminPassword)\n .set(\"neo4j.db.transaction.timeout\", \"1000\")\n val session = SparkSession.builder.config(newConf).getOrCreate()\n\n val cypher = \"UNWIND range(1, 20) AS i \" +\n \"CALL apoc.util.sleep(1000) \" +\n \"RETURN i as number\"\n\n val df = session.read.format(\"org.neo4j.spark.DataSource\")\n .option(\"query\", cypher)\n\n val exc = assertThrows(\n classOf[ClientException],\n () => {\n df.load()\n .toDF()\n .select(\"number\").rdd.map(_.getLong(0)).collect().toList\n }\n )\n assertTrue(exc.getMessage.contains(\"The transaction has been terminated\"))\n }\n\n @Test\n def sparkConnectorFailsWithTransactionTimeoutWhenSetOnDatasourceLevel(): Unit = {\n val newConf = conf.clone().set(\"neo4j.url\", server.getBoltUrl)\n .set(\"neo4j.authentication.basic.username\", \"neo4j\")\n .set(\"neo4j.authentication.basic.password\", server.getAdminPassword)\n val session = SparkSession.builder.config(newConf).getOrCreate()\n\n val cypher = \"UNWIND range(1, 20) AS i \" +\n \"CALL apoc.util.sleep(1000) \" +\n \"RETURN i as number\"\n\n val df = session.read.format(\"org.neo4j.spark.DataSource\")\n .option(\"query\", cypher)\n .option(\"db.transaction.timeout\", \"1000\")\n\n val exc = assertThrows(\n classOf[ClientException],\n () => {\n df.load()\n .toDF()\n .select(\"number\").rdd.map(_.getLong(0)).collect().toList\n }\n )\n assertTrue(exc.getMessage.contains(\"The transaction has been terminated\"))\n }\n\n @Test\n def sparkConnectorExtendsDefaultTimeout(): Unit = {\n val cypher = \"UNWIND range(1, 6) AS i \" +\n \"CALL apoc.util.sleep(2000) \" +\n \"RETURN i as number\"\n val df = ss.read.format(\"org.neo4j.spark.DataSource\")\n .option(\"url\", NEO4J_LOW_TX_TIMEOUT.getBoltUrl)\n .option(\"authentication.basic.username\", \"neo4j\")\n .option(\"authentication.basic.password\", NEO4J_LOW_TX_TIMEOUT.getAdminPassword)\n .option(\"db.transaction.timeout\", \"15000\")\n .option(\"query\", cypher)\n .load()\n .toDF()\n\n val results = df.select(\"number\").rdd.map(_.getLong(0)).collect().toList\n val expected = Range.inclusive(1, 6).map(_.toLong).toList\n\n assertEquals(expected, results)\n }\n\n}\n\nobject TransactionTimeoutIT {\n\n private val NEO4J_LOW_TX_TIMEOUT = new Neo4jContainerExtension {\n withNeo4jConfig(\"dbms.security.auth_enabled\", \"false\")\n withEnv(\"NEO4J_ACCEPT_LICENSE_AGREEMENT\", \"yes\")\n withEnv(\"NEO4JLABS_PLUGINS\", \"[\\\"apoc\\\"]\")\n withEnv(\"NEO4J_db_temporal_timezone\", TimeZone.getDefault.getID)\n withNeo4jConfig(\"db.transaction.timeout\", \"10s\")\n withDatabases(Seq(\"db1\", \"db2\"))\n }\n\n @BeforeClass\n def setUp(): Unit = {\n NEO4J_LOW_TX_TIMEOUT.start()\n }\n\n @AfterClass\n def tearDown() = {\n TestUtil.closeSafely(NEO4J_LOW_TX_TIMEOUT)\n }\n}\n" }, { "path": "test-support/pom.xml", "content": "\n\n 4.0.0\n \n org.neo4j\n neo4j-connector-apache-spark_parent\n 5.4.3-SNAPSHOT\n \n neo4j-connector-apache-spark_test-support\n jar\n neo4j-connector-apache-spark-test-support\n Test Utilities for Neo4j Connector for Apache Spark using the binary Bolt Driver\n \n true\n 2.0.9\n \n \n \n \n org.objenesis\n objenesis\n 3.5\n \n \n \n \n \n junit\n junit\n \n \n org.hamcrest\n hamcrest\n \n \n org.neo4j\n caniuse-core\n \n \n org.neo4j\n caniuse-neo4j-detection\n \n \n org.neo4j.driver\n neo4j-java-driver-slim\n \n \n org.powermock\n powermock-api-mockito2\n ${powermock.version}\n \n \n org.powermock\n powermock-module-junit4\n ${powermock.version}\n \n \n org.scalatest\n scalatest_${scala.binary.version}\n \n \n org.scalatestplus\n junit-4-13_${scala.binary.version}\n \n \n org.testcontainers\n testcontainers\n \n \n org.jetbrains\n annotations\n \n \n \n \n org.testcontainers\n testcontainers-neo4j\n \n \n org.apache.spark\n spark-core_${scala.binary.version}\n provided\n \n \n org.apache.spark\n spark-sql_${scala.binary.version}\n provided\n \n \n \n \n \n net.alchim31.maven\n scala-maven-plugin\n \n \n org.apache.maven.plugins\n maven-failsafe-plugin\n \n \n org.apache.maven.plugins\n maven-surefire-plugin\n \n \n org.apache.maven.plugins\n maven-deploy-plugin\n \n true\n \n \n \n \n\n" }, { "path": "test-support/src/main/java/org/neo4j/spark/Assert.java", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark;//\n// Source code recreated from a .class file by IntelliJ IDEA\n// (powered by Fernflower decompiler)\n//\n\nimport java.util.Arrays;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Function;\nimport java.util.function.Supplier;\n\nimport org.hamcrest.Description;\nimport org.hamcrest.Matcher;\nimport org.hamcrest.MatcherAssert;\nimport org.hamcrest.StringDescription;\nimport org.hamcrest.core.StringContains;\n\npublic final class Assert {\n private Assert() {\n }\n\n public interface ThrowingSupplier {\n T get() throws E;\n\n static ThrowingSupplier throwingSupplier(final Supplier supplier) {\n return new ThrowingSupplier() {\n public TYPE get() {\n return supplier.get();\n }\n\n public String toString() {\n return supplier.toString();\n }\n };\n }\n }\n\n public interface ThrowingAction {\n void apply() throws E;\n\n static ThrowingAction noop() {\n return () -> {\n };\n }\n }\n\n public static void assertException(ThrowingAction f, Class typeOfException) {\n assertException(f, typeOfException, (String) null);\n }\n\n public static void assertException(ThrowingAction f, Class typeOfException, String partOfErrorMessage) {\n try {\n f.apply();\n org.junit.Assert.fail(\"Expected exception of type \" + typeOfException + \", but no exception was thrown\");\n } catch (Exception var4) {\n if (typeOfException.isInstance(var4)) {\n if (partOfErrorMessage != null) {\n MatcherAssert.assertThat(var4.getMessage(), StringContains.containsString(partOfErrorMessage));\n }\n } else {\n org.junit.Assert.fail(\"Got unexpected exception \" + var4.getClass() + \"\\nExpected: \" + typeOfException);\n }\n }\n\n }\n\n public static void assertEventually(ThrowingSupplier actual, Matcher matcher, long timeout, TimeUnit timeUnit) throws E, InterruptedException {\n assertEventually((ignored) -> {\n return \"\";\n }, actual, matcher, timeout, timeUnit);\n }\n\n public static void assertEventually(String reason, ThrowingSupplier actual, Matcher matcher, long timeout, TimeUnit timeUnit) throws E, InterruptedException {\n assertEventually((ignored) -> {\n return reason;\n }, actual, matcher, timeout, timeUnit);\n }\n\n public static void assertEventually(Function reason, ThrowingSupplier actual, Matcher matcher, long timeout, TimeUnit timeUnit) throws E, InterruptedException {\n long endTimeMillis = System.currentTimeMillis() + timeUnit.toMillis(timeout);\n\n while (true) {\n long sampleTime = System.currentTimeMillis();\n T last = actual.get();\n boolean matched = matcher.matches(last);\n if (matched || sampleTime > endTimeMillis) {\n if (!matched) {\n Description description = new StringDescription();\n description.appendText((String) reason.apply(last)).appendText(\"\\nExpected: \").appendDescriptionOf(matcher).appendText(\"\\n but: \");\n matcher.describeMismatch(last, description);\n throw new AssertionError(\"Timeout hit (\" + timeout + \" \" + timeUnit.toString().toLowerCase() + \") while waiting for condition to match: \" + description.toString());\n } else {\n return;\n }\n }\n\n Thread.sleep(100L);\n }\n }\n\n private static AssertionError newAssertionError(String message, Object expected, Object actual) {\n return new AssertionError((message != null && !message.isEmpty() ? message + \"\\n\" : \"\") + \"Expected: \" + prettyPrint(expected) + \", actual: \" + prettyPrint(actual));\n }\n\n private static String prettyPrint(Object o) {\n if (o == null) {\n return \"null\";\n }\n\n Class clazz = o.getClass();\n if (clazz.isArray()) {\n if (clazz == byte[].class) {\n return Arrays.toString((byte[]) o);\n } else if (clazz == short[].class) {\n return Arrays.toString((short[]) o);\n } else if (clazz == int[].class) {\n return Arrays.toString((int[]) o);\n } else if (clazz == long[].class) {\n return Arrays.toString((long[]) o);\n } else if (clazz == float[].class) {\n return Arrays.toString((float[]) o);\n } else if (clazz == double[].class) {\n return Arrays.toString((double[]) o);\n } else if (clazz == char[].class) {\n return Arrays.toString((char[]) o);\n } else if (clazz == boolean[].class) {\n return Arrays.toString((boolean[]) o);\n } else {\n return Arrays.deepToString((Object[]) o);\n }\n } else {\n return String.valueOf(o);\n }\n }\n}\n\n" }, { "path": "test-support/src/main/resources/simplelogger.properties", "content": "org.slf4j.simpleLogger.defaultLogLevel=error\norg.slf4j.simpleLogger.showDateTime=true\norg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd'T'HH:mm:ss.SSS\n" }, { "path": "test-support/src/main/scala/org/neo4j/Closeables.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j\n\nobject Closeables {\n\n def use[A <: AutoCloseable, B](resource: A)(code: A ⇒ B): B =\n try\n code(resource)\n finally\n resource.close()\n\n}\n" }, { "path": "test-support/src/main/scala/org/neo4j/Neo4jContainerExtension.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j\n\nimport org.neo4j.driver.AuthToken\nimport org.neo4j.driver.AuthTokens\nimport org.neo4j.driver.GraphDatabase\nimport org.neo4j.driver.SessionConfig\nimport org.neo4j.spark.TestUtil\nimport org.rnorth.ducttape.unreliables.Unreliables\nimport org.testcontainers.containers.wait.strategy.AbstractWaitStrategy\nimport org.testcontainers.containers.wait.strategy.WaitAllStrategy\nimport org.testcontainers.neo4j.Neo4jContainer\n\nimport java.time.Duration\nimport java.util.concurrent.Callable\nimport java.util.concurrent.TimeUnit\n\nimport scala.collection.JavaConverters._\nimport scala.io.Source\n\nclass DatabasesWaitStrategy(private val auth: AuthToken) extends AbstractWaitStrategy {\n private var databases = Seq.empty[String]\n\n def forDatabases(dbs: Seq[String]): DatabasesWaitStrategy = {\n databases ++= dbs\n this\n }\n\n override def waitUntilReady(): Unit = {\n val boltUrl = s\"bolt://${waitStrategyTarget.getHost}:${waitStrategyTarget.getMappedPort(7687)}\"\n val driver = GraphDatabase.driver(boltUrl, auth)\n try {\n Unreliables.retryUntilSuccess(\n startupTimeout.getSeconds.toInt,\n TimeUnit.SECONDS,\n new Callable[Unit] {\n override def call(): Unit = {\n val session = driver.session(SessionConfig.forDatabase(\"system\"))\n try {\n databases.foreach { db =>\n session.writeTransaction(tx => tx.run(s\"CREATE DATABASE $db IF NOT EXISTS WAIT 30 SECONDS\").consume())\n\n val status = session.readTransaction(tx => {\n tx.run(s\"SHOW DATABASE $db YIELD currentStatus\").single().get(\"currentStatus\").asString()\n })\n if (status != \"online\") {\n throw new RuntimeException(s\"Database $db is not online yet, current status: $status\")\n }\n }\n } finally {\n session.close()\n }\n }\n }\n )\n } finally {\n driver.close()\n }\n }\n}\n\n// docker pull neo4j/neo4j-experimental:4.0.0-rc01-enterprise\nclass Neo4jContainerExtension\n extends Neo4jContainer(\n TestUtil.neo4jImage()\n ) {\n private var databases: Seq[String] = Seq.empty\n\n private var fixture: Set[(String, String)] = Set.empty\n\n def withDatabases(dbs: Seq[String]): Neo4jContainerExtension = {\n databases ++= dbs\n this\n }\n\n def withFixture(database: String, path: String): Neo4jContainerExtension = {\n fixture ++= Set((database, path))\n this\n }\n\n private def createAuth(): AuthToken =\n if (getAdminPassword.nonEmpty) AuthTokens.basic(\"neo4j\", getAdminPassword) else AuthTokens.none()\n\n override def start(): Unit = {\n if (databases.nonEmpty) {\n val waitAllStrategy = waitStrategy.asInstanceOf[WaitAllStrategy]\n waitAllStrategy.withStrategy(\n new DatabasesWaitStrategy(createAuth()).forDatabases(databases).withStartupTimeout(Duration.ofMinutes(2))\n )\n }\n addEnv(\"NEO4J_ACCEPT_LICENSE_AGREEMENT\", \"yes\")\n super.start()\n\n if (fixture.nonEmpty) {\n val driver = GraphDatabase.driver(this.getBoltUrl, createAuth())\n try {\n fixture.foreach(t => {\n val session = driver.session(SessionConfig.forDatabase(t._1))\n try {\n val lines = Source.fromResource(t._2)\n .mkString(\"\\n\")\n .split(\";\")\n lines.foreach(line => session.run(line))\n } finally {\n TestUtil.closeSafely(session)\n }\n })\n } finally {\n TestUtil.closeSafely(driver)\n }\n }\n }\n}\n" }, { "path": "test-support/src/main/scala/org/neo4j/spark/RowUtil.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.sql.Row\n\nobject RowUtil {\n def getByName[T](row: Row, name: String): T = row.getAs[T](row.fieldIndex(name))\n}\n" }, { "path": "test-support/src/main/scala/org/neo4j/spark/SparkConnectorScalaBaseTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.SparkConf\nimport org.apache.spark.sql.SparkSession\nimport org.junit._\nimport org.junit.rules.TestName\nimport org.neo4j.Closeables.use\nimport org.scalatestplus.junit.AssertionsForJUnit\n\nimport scala.annotation.meta.getter\n\nobject SparkConnectorScalaBaseTSE {\n\n private var startedFromSuite = true\n\n @BeforeClass\n def setUpContainer() = {\n if (!SparkConnectorScalaSuiteIT.server.isRunning) {\n startedFromSuite = false\n SparkConnectorScalaSuiteIT.setUpContainer()\n }\n }\n\n @AfterClass\n def tearDownContainer() = {\n if (!startedFromSuite) {\n SparkConnectorScalaSuiteIT.tearDownContainer()\n }\n }\n\n}\n\nclass SparkConnectorScalaBaseTSE extends AssertionsForJUnit {\n val conf: SparkConf = SparkConnectorScalaSuiteIT.conf\n val ss: SparkSession = SparkConnectorScalaSuiteIT.ss\n\n @(Rule @getter)\n val testName: TestName = new TestName\n\n @Before\n def before(): Unit = {\n use(SparkConnectorScalaSuiteIT.session(\"system\")) { session =>\n session\n .run(\"CREATE OR REPLACE DATABASE neo4j WAIT 30 seconds\").consume()\n }\n }\n\n @After\n def after(): Unit = {\n ss.catalog.listTables()\n .collect()\n .foreach(t => ss.catalog.dropTempView(t.name))\n ss.catalog.listTables()\n .collect()\n .foreach(t => ss.catalog.dropGlobalTempView(t.name))\n }\n}\n" }, { "path": "test-support/src/main/scala/org/neo4j/spark/SparkConnectorScalaBaseWithApocTSE.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.SparkConf\nimport org.apache.spark.sql.SparkSession\nimport org.hamcrest.Matchers\nimport org.junit._\nimport org.junit.rules.TestName\nimport org.neo4j.Closeables.use\nimport org.neo4j.driver.Transaction\nimport org.neo4j.driver.TransactionWork\nimport org.neo4j.driver.summary.ResultSummary\nimport org.neo4j.spark\n\nimport java.util.concurrent.TimeUnit\n\nimport scala.annotation.meta.getter\n\nobject SparkConnectorScalaBaseWithApocTSE {\n\n private var startedFromSuite = true\n\n @BeforeClass\n def setUpContainer() = {\n if (!SparkConnectorScalaSuiteWithApocIT.server.isRunning) {\n startedFromSuite = false\n SparkConnectorScalaSuiteWithApocIT.setUpContainer()\n }\n }\n\n @AfterClass\n def tearDownContainer() = {\n if (!startedFromSuite) {\n SparkConnectorScalaSuiteWithApocIT.tearDownContainer()\n }\n }\n\n}\n\nclass SparkConnectorScalaBaseWithApocTSE {\n\n val conf: SparkConf = SparkConnectorScalaSuiteWithApocIT.conf\n val ss: SparkSession = SparkConnectorScalaSuiteWithApocIT.ss\n\n @(Rule @getter)\n val testName: TestName = new TestName\n\n @Before\n def before() {\n use(SparkConnectorScalaSuiteWithApocIT.session(\"system\")) {\n session =>\n session.run(\"CREATE OR REPLACE DATABASE neo4j WAIT 30 seconds\")\n .consume()\n }\n }\n\n}\n" }, { "path": "test-support/src/main/scala/org/neo4j/spark/SparkConnectorScalaSuiteIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.SparkConf\nimport org.apache.spark.sql.SparkSession\nimport org.junit.AfterClass\nimport org.junit.Assume\nimport org.junit.BeforeClass\nimport org.neo4j.Neo4jContainerExtension\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.caniuse.Neo4jDetector\nimport org.neo4j.driver._\n\nimport java.io.File\nimport java.nio.file.Files\nimport java.util.TimeZone\n\nobject SparkConnectorScalaSuiteIT {\n\n val server: Neo4jContainerExtension = new Neo4jContainerExtension {\n withNeo4jConfig(\"dbms.security.auth_enabled\", \"false\")\n withEnv(\"NEO4J_ACCEPT_LICENSE_AGREEMENT\", \"yes\")\n withEnv(\"NEO4J_db_temporal_timezone\", TimeZone.getDefault.getID)\n withDatabases(Seq(\"db1\", \"db2\"))\n }\n\n var conf: SparkConf = _\n var ss: SparkSession = _\n var driver: Driver = _\n var tmpDir: File = _\n var neo4j: Neo4j = _\n\n @BeforeClass\n def setUpContainer(): Unit = {\n if (!server.isRunning) {\n try {\n server.start()\n } catch {\n case _: Throwable => //\n }\n Assume.assumeTrue(\"Neo4j container is not started\", server.isRunning)\n tmpDir = Files.createTempDirectory(\"spark-warehouse\").toFile\n tmpDir.deleteOnExit()\n conf = new SparkConf()\n .setAppName(\"neoTest\")\n .setMaster(\"local[*]\")\n .set(\"spark.driver.host\", \"127.0.0.1\")\n .set(\"spark.sql.warehouse.dir\", tmpDir.getAbsolutePath)\n ss = SparkSession.builder.config(conf).getOrCreate()\n driver = GraphDatabase.driver(server.getBoltUrl, AuthTokens.none())\n neo4j = Neo4jDetector.INSTANCE.detect(driver)\n }\n }\n\n @AfterClass\n def tearDownContainer() = {\n TestUtil.closeSafely(driver)\n TestUtil.closeSafely(server)\n TestUtil.closeSafely(ss)\n }\n\n def session(database: String = \"\"): Session = {\n if (database.isEmpty) {\n driver.session()\n } else {\n driver.session(SessionConfig.forDatabase(database))\n }\n }\n}\n\nclass SparkConnectorScalaSuiteIT {}\n" }, { "path": "test-support/src/main/scala/org/neo4j/spark/SparkConnectorScalaSuiteWithApocIT.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.SparkConf\nimport org.apache.spark.sql.SparkSession\nimport org.junit.AfterClass\nimport org.junit.Assume\nimport org.junit.BeforeClass\nimport org.neo4j.Neo4jContainerExtension\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.caniuse.Neo4jDetector\nimport org.neo4j.driver._\n\nimport java.util.TimeZone\n\nobject SparkConnectorScalaSuiteWithApocIT {\n\n val server: Neo4jContainerExtension = new Neo4jContainerExtension {\n withNeo4jConfig(\"dbms.security.auth_enabled\", \"false\")\n withEnv(\"NEO4J_ACCEPT_LICENSE_AGREEMENT\", \"yes\")\n withEnv(\"NEO4JLABS_PLUGINS\", \"[\\\"apoc\\\"]\")\n withEnv(\"NEO4J_db_temporal_timezone\", TimeZone.getDefault.getID)\n withDatabases(Seq(\"db1\", \"db2\"))\n }\n\n var conf: SparkConf = _\n var ss: SparkSession = _\n var driver: Driver = _\n var neo4j: Neo4j = _\n\n @BeforeClass\n def setUpContainer(): Unit = {\n if (!server.isRunning) {\n try {\n server.start()\n } catch {\n case _: Throwable => //\n }\n Assume.assumeTrue(\"Neo4j container is not started\", server.isRunning)\n conf = new SparkConf()\n .setAppName(\"neoTest\")\n .setMaster(\"local[*]\")\n .set(\"spark.driver.host\", \"127.0.0.1\")\n ss = SparkSession.builder.config(conf).getOrCreate()\n driver = GraphDatabase.driver(server.getBoltUrl, AuthTokens.none())\n neo4j = Neo4jDetector.INSTANCE.detect(driver)\n }\n Assume.assumeTrue(\"Neo4j Preview versions doesn't have APOC\", TestUtil.hasApoc(session()))\n }\n\n @AfterClass\n def tearDownContainer() = {\n TestUtil.closeSafely(driver)\n TestUtil.closeSafely(server)\n TestUtil.closeSafely(ss)\n }\n\n def session(database: String = \"\"): Session = {\n if (database.isEmpty) {\n driver.session()\n } else {\n driver.session(SessionConfig.forDatabase(database))\n }\n }\n}\n\nclass SparkConnectorScalaSuiteWithApocIT {}\n" }, { "path": "test-support/src/main/scala/org/neo4j/spark/SparkConnectorScalaSuiteWithGdsBase.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.apache.spark.SparkConf\nimport org.apache.spark.sql.SparkSession\nimport org.junit.AfterClass\nimport org.junit.Assume\nimport org.junit.Before\nimport org.junit.BeforeClass\nimport org.junit.Rule\nimport org.junit.rules.TestName\nimport org.neo4j.Closeables.use\nimport org.neo4j.Neo4jContainerExtension\nimport org.neo4j.caniuse.Neo4j\nimport org.neo4j.caniuse.Neo4jDetector\nimport org.neo4j.driver._\n\nimport java.util.TimeZone\n\nimport scala.annotation.meta.getter\n\nobject SparkConnectorScalaSuiteWithGdsBase {\n\n val server: Neo4jContainerExtension = new Neo4jContainerExtension {\n withNeo4jConfig(\"dbms.security.auth_enabled\", \"false\")\n withEnv(\"NEO4J_ACCEPT_LICENSE_AGREEMENT\", \"yes\")\n withEnv(\"NEO4JLABS_PLUGINS\", \"[\\\"graph-data-science\\\"]\")\n withEnv(\"NEO4J_db_temporal_timezone\", TimeZone.getDefault.getID)\n withDatabases(Seq(\"db1\", \"db2\"))\n }\n\n var conf: SparkConf = _\n var ss: SparkSession = _\n var driver: Driver = _\n var neo4j: Neo4j = _\n\n @BeforeClass\n def setUpContainer(): Unit = {\n if (!server.isRunning) {\n try {\n server.start()\n } catch {\n case _: Throwable => //\n }\n Assume.assumeTrue(\"Neo4j container is not started\", server.isRunning)\n conf = new SparkConf()\n .setAppName(\"neoTest\")\n .setMaster(\"local[*]\")\n .set(\"spark.driver.host\", \"127.0.0.1\")\n ss = SparkSession.builder.config(conf).getOrCreate()\n driver = GraphDatabase.driver(server.getBoltUrl, AuthTokens.none())\n neo4j = Neo4jDetector.INSTANCE.detect(driver)\n }\n }\n\n @AfterClass\n def tearDownContainer(): Unit = {\n TestUtil.closeSafely(driver)\n TestUtil.closeSafely(server)\n TestUtil.closeSafely(ss)\n }\n\n def session(database: String = \"\"): Session = {\n if (database.isEmpty) {\n driver.session()\n } else {\n driver.session(SessionConfig.forDatabase(database))\n }\n }\n}\n\nclass SparkConnectorScalaSuiteWithGdsBase {\n\n val conf: SparkConf = SparkConnectorScalaSuiteWithGdsBase.conf\n val ss: SparkSession = SparkConnectorScalaSuiteWithGdsBase.ss\n\n @(Rule @getter)\n val testName: TestName = new TestName\n\n @Before\n def before(): Unit = {\n use(SparkConnectorScalaSuiteWithGdsBase.session(\"system\")) {\n session =>\n session.run(\"CREATE OR REPLACE DATABASE neo4j WAIT 30 seconds\")\n .consume()\n }\n }\n}\n" }, { "path": "test-support/src/main/scala/org/neo4j/spark/TestUtil.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.neo4j.driver.Session\nimport org.neo4j.driver.Transaction\nimport org.slf4j.Logger\nimport org.testcontainers.utility.DockerImageName\n\nimport java.util.Properties\n\ncase class Version(major: Int, minor: Int, patch: Int) {\n\n override def toString: String = s\"${major}.${minor}.${patch}\"\n}\n\nobject Version {\n implicit val ordering: Ordering[Version] = Ordering.by(v => (v.major, v.minor, v.patch))\n\n def parse(version: String): Version = {\n val fields = version.split(\"\\\\.\")\n .map(field => {\n val additionalInfoIndex = field.indexOf('-')\n if (additionalInfoIndex.equals(-1)) field\n else field.substring(0, additionalInfoIndex)\n })\n .map(_.toInt)\n .toList\n\n Version(fields.head, fields(1), fields(2))\n }\n\n}\n\nobject Versions {\n\n val GDS_2_4: Version = Version(2, 4, 0)\n val GDS_2_5: Version = Version(2, 5, 0)\n val NEO4J_4_4: Version = Version(4, 4, 0)\n val NEO4J_5: Version = Version(5, 0, 0)\n val NEO4J_5_13: Version = Version(5, 13, 0)\n\n}\n\nobject TestUtil {\n\n def neo4jImage(): DockerImageName = {\n val image = Option(System.getenv(\"NEO4J_TEST_IMAGE\"))\n .map(_.trim)\n .filter(_.nonEmpty) // avoids Java 11-only isBlank\n .getOrElse(throw new IllegalArgumentException(\"NEO4J_TEST_IMAGE environment variable is not defined!\"))\n DockerImageName.parse(image).asCompatibleSubstituteFor(\"neo4j\")\n }\n\n def gdsVersion(session: Session): Version = {\n Version.parse(session.run(\n \"CALL gds.debug.sysInfo() YIELD key, value WHERE key = 'gdsVersion' RETURN value\"\n ).single().get(0).asString())\n }\n\n def neo4jVersion(session: Session): Version = {\n Version.parse(session.run(\n \"CALL dbms.components() YIELD name, versions WHERE name = 'Neo4j Kernel' RETURN versions[0]\"\n ).single().get(0).asString())\n }\n\n def hasApoc(session: Session): Boolean = {\n val result = session.run(\n \"SHOW PROCEDURES YIELD name RETURN any(x IN collect(name) WHERE x STARTS WITH 'apoc.') AS hasApoc\"\n )\n result.single().get(\"hasApoc\").asBoolean()\n }\n\n def closeSafely(autoCloseable: AutoCloseable, logger: Logger = null): Unit = {\n try {\n autoCloseable match {\n case s: Session => if (s.isOpen) s.close()\n case t: Transaction => if (t.isOpen) t.close()\n case null => ()\n case _ => autoCloseable.close()\n }\n } catch {\n case t: Throwable => if (logger != null) {\n t.printStackTrace()\n logger.warn(s\"Cannot close ${autoCloseable.getClass.getSimpleName} because of the following exception:\", t)\n }\n }\n }\n\n}\n" }, { "path": "test-support/src/test/scala/org/neo4j/spark/VersionTest.scala", "content": "/*\n * Copyright (c) \"Neo4j\"\n * Neo4j Sweden AB [https://neo4j.com]\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.neo4j.spark\n\nimport org.junit.Assert.assertEquals\nimport org.junit.Test\n\nclass VersionTest {\n\n @Test\n def parses_versions(): Unit = {\n assertEquals(Version(5, 26, 399), Version.parse(\"5.26.399\"))\n assertEquals(Version(2025, 11, 0), Version.parse(\"2025.11.0-41865\"))\n }\n}\n" } ]