[ { "path": ".github/workflows/ci-cd.yml", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\nname: \"CI/CD Pipeline\"\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n release:\n types: [ published ]\n workflow_dispatch:\n inputs:\n tag:\n description: \"Tag to publish in addition to `latest`\"\n required: true\n type: string\n\njobs:\n # Define the matrix once for all jobs\n setup:\n runs-on: ubuntu-24.04\n outputs:\n matrix: ${{ steps.set-matrix.outputs.matrix }}\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n \n - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5\n with:\n distribution: \"temurin\"\n java-version: \"21\"\n cache: \"maven\"\n\n - name: Resolve all dependencies\n # go-offline ensures the .m2 folder is full before the matrix starts\n run: |\n mvn dependency:go-offline -B --no-transfer-progress \\\n -Dmaven.wagon.http.retryHandler.count=3 \\\n -Dmaven.wagon.http.connectionTimeout=120000 \\\n -Dmaven.wagon.http.readTimeout=120000 \\\n -Dhttp.retryHandler.count=5\n\n - id: set-matrix\n run: |\n # Define configuration for all storage variants\n STRATEGY='{\n \"include\": [\n {\n \"variant\": \"cassandra\",\n \"storage\": \"cassandra\",\n \"es_version\": \"\",\n \"image_tag\": \"test-cassandra\",\n \"es_test_ver\": \"\"\n },\n {\n \"variant\": \"elasticsearch7\",\n \"storage\": \"elasticsearch\",\n \"es_version\": \"7.17.29\",\n \"image_tag\": \"test-es7\",\n \"es_test_ver\": \"7.3.0\"\n },\n {\n \"variant\": \"elasticsearch8\",\n \"storage\": \"elasticsearch\",\n \"es_version\": \"8.13.4\",\n \"image_tag\": \"test-es8\",\n \"es_test_ver\": \"8.13.4\"\n },\n {\n \"variant\": \"elasticsearch9\",\n \"storage\": \"elasticsearch\",\n \"es_version\": \"9.1.3\",\n \"image_tag\": \"test-es9\",\n \"es_test_ver\": \"9.1.3\"\n },\n {\n \"variant\": \"opensearch\",\n \"storage\": \"opensearch\",\n \"es_version\": \"\",\n \"image_tag\": \"test-opensearch\",\n \"es_test_ver\": \"2.14.0\"\n }\n ]\n }'\n # Convert to a single line and output\n echo \"matrix=$(echo $STRATEGY | jq -c .)\" >> $GITHUB_OUTPUT\n\n build-jars:\n name: Build JAR - ${{ matrix.variant }}\n runs-on: ubuntu-24.04\n needs: setup\n strategy:\n fail-fast: false\n matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Set up JDK 21\n uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5\n with:\n distribution: \"temurin\"\n java-version: \"21\"\n cache: \"maven\"\n\n - name: Build JAR\n env:\n MAVEN_ES_PROP: ${{ matrix.es_version != '' && format('-Dversion.elasticsearch.spark={0}', matrix.es_version) || '' }}\n run: |\n ./mvnw clean package --batch-mode --no-transfer-progress -Dlicense.skip=true -DskipTests \\\n -pl jaeger-spark-dependencies-${{ matrix.storage }} -am \\\n $MAVEN_ES_PROP\n\n - name: Prepare artifact\n run: |\n mkdir -p artifact-target\n cp jaeger-spark-dependencies-${{ matrix.storage }}/target/jaeger-spark-dependencies-${{ matrix.storage }}-0.0.1-SNAPSHOT.jar \\\n artifact-target/\n\n - name: Upload JAR artifact\n uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4\n with:\n name: jar-${{ matrix.variant }}\n path: artifact-target/*.jar\n retention-days: 1\n\n e2e-tests:\n name: E2E Tests - ${{ matrix.variant }}\n runs-on: ubuntu-24.04\n needs: [setup, build-jars]\n strategy:\n fail-fast: false\n matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Set up JDK 21\n uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5\n with:\n distribution: \"temurin\"\n java-version: \"21\"\n cache: \"maven\"\n\n - name: Download JAR artifact\n uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4\n with:\n name: jar-${{ matrix.variant }}\n path: artifact-target/\n\n - name: Build local Docker image\n uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2\n with:\n context: .\n load: true\n push: false\n tags: ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:${{ matrix.image_tag }}\n build-args: |\n VARIANT=${{ matrix.variant }}\n\n - name: Run integration tests\n env:\n SPARK_DEPENDENCIES_JOB_IMAGE_TAG: ${{ matrix.image_tag }}\n ELASTICSEARCH_VERSION: ${{ matrix.es_test_ver }}\n OPENSEARCH_VERSION: ${{ matrix.es_test_ver }}\n # The es_spark version is only needed when testing Elasticsearch variants\n MAVEN_ES_PROP: ${{ matrix.es_version != '' && format('-Dversion.elasticsearch.spark={0}', matrix.es_version) || '' }}\n run: |\n for attempt in 1 2 3; do\n echo \"Integration test attempt $attempt of 3\"\n ./mvnw --batch-mode --no-transfer-progress test -am \\\n -pl jaeger-spark-dependencies-${{ matrix.storage }} \\\n $MAVEN_ES_PROP && break\n if [ $attempt -lt 3 ]; then\n echo \"Attempt $attempt failed, retrying after 15 seconds...\"\n sleep 15\n else\n echo \"All attempts failed\"\n exit 1\n fi\n done\n\n publish:\n name: Publish - ${{ matrix.variant }}\n runs-on: ubuntu-24.04\n needs: [setup, e2e-tests]\n strategy:\n fail-fast: false\n matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}\n steps:\n - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2\n\n - name: Download JAR artifact\n uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4\n with:\n name: jar-${{ matrix.variant }}\n path: artifact-target/\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0\n with:\n install: true\n platforms: |\n linux/amd64\n linux/arm64\n\n - name: Compute Tags\n id: compute-tags\n run: |\n prefix=\"ghcr.io/jaegertracing/spark-dependencies/spark-dependencies\"\n variant=\"${{ matrix.variant }}\"\n \n # For main releases, use variant suffix\n if [[ \"${{ github.event_name }}\" == \"release\" ]] && [[ \"${{ github.ref }}\" == refs/tags/* ]]; then\n release=$(echo ${{ github.ref }} | sed 's/refs\\/tags\\///g')\n tags=\"$prefix:$release-$variant\"\n \n # elasticsearch9 gets the 'latest' tag as it supports the newest ES version\n if [[ \"$variant\" == \"elasticsearch9\" ]]; then\n tags=\"$tags,$prefix:latest\"\n fi\n elif [[ -n \"${{ inputs.tag }}\" ]]; then\n tags=\"$prefix:${{ inputs.tag }}-$variant\"\n else\n # For main branch builds, use main-variant\n tags=\"$prefix:main-$variant\"\n fi\n\n echo \"Computed tags for publication ($variant): $tags\"\n echo \"tags=$tags\" >> $GITHUB_OUTPUT\n\n - name: Login to GitHub Package Registry\n uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0\n if: github.event_name != 'pull_request'\n with:\n registry: ghcr.io\n username: ${{ github.repository_owner }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Build and push multi-arch images\n uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2\n with:\n context: .\n push: ${{ github.event_name != 'pull_request' }}\n tags: ${{ steps.compute-tags.outputs.tags }}\n platforms: |\n linux/amd64\n linux/arm64\n build-args: |\n VARIANT=${{ matrix.variant }}\n" }, { "path": ".github/workflows/stale.yml", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\nname: 'Close stale issues and PRs'\n\non:\n schedule:\n # Run every Monday at 1:30 AM UTC\n - cron: '30 1 * * 1'\n workflow_dispatch:\n\npermissions:\n issues: write\n pull-requests: write\n\njobs:\n stale:\n runs-on: ubuntu-latest\n steps:\n - name: Harden Runner\n uses: step-security/harden-runner@e3f713f2d8f53843e71c69a996d56f51aa9adfb9\n with:\n egress-policy: audit\n\n - uses: actions/stale@dcd2b9469d2220b7e8d08aedc00c105d277fd46b\n with:\n # Issues configuration\n days-before-issue-stale: 90\n days-before-issue-close: 14\n stale-issue-message: >\n This issue has been automatically marked as stale because it has not had\n recent activity. It will be closed if no further activity occurs.\n To keep it open either add a comment or the label `do-not-expire`.\n close-issue-message: >\n This issue has been automatically closed due to inactivity.\n stale-issue-label: 'stale'\n exempt-issue-labels: 'do-not-expire,help-wanted'\n only-issue-labels: 'question'\n \n # Pull requests configuration \n days-before-pr-stale: 60\n days-before-pr-close: 14\n stale-pr-message: >\n This pull request has been automatically marked as stale because it has not had\n recent activity. It will be closed if no further activity occurs. You may re-open\n it if you need more time.\n close-pr-message: >\n This pull request has been automatically closed due to inactivity. You may re-open\n it if you need more time. We really appreciate your contribution and we are sorry\n that this has not been completed.\n stale-pr-label: 'stale'\n exempt-pr-labels: 'do-not-expire'\n \n # General configuration\n operations-per-run: 100\n remove-stale-when-updated: true\n" }, { "path": ".gitignore", "content": "target/\npom.xml.tag\npom.xml.releaseBackup\npom.xml.versionsBackup\npom.xml.next\nrelease.properties\ndependency-reduced-pom.xml\nbuildNumber.properties\n.mvn/timing.properties\n.*.swp\n*.iml\n.idea\n\n# Avoid ignoring Maven wrapper jar file (.jar files are usually ignored)\n!/.mvn/wrapper/maven-wrapper.jar\n\n.testcontainers-tmp-*\n.vscode/\nartifact-target/\n" }, { "path": ".mvn/wrapper/maven-wrapper.properties", "content": "distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.12/apache-maven-3.9.12-bin.zip\n" }, { "path": "AGENTS.md", "content": "# Integration Tests Guide\n\nThis guide provides instructions for running integration tests for the Jaeger Spark Dependencies project.\n\nFor detailed information about integration tests, including prerequisites, troubleshooting, and environment variables, see the [Running Integration Tests](README.md#running-integration-tests) section in the README.\n\n## Quick Start\n\nThe project includes make targets for running integration tests against different storage backends:\n\n```bash\nmake e2e-cassandra # Run Cassandra 4.x integration tests\nmake e2e-es7 # Run Elasticsearch 7 integration tests\nmake e2e-es8 # Run Elasticsearch 8 integration tests\nmake e2e-es9 # Run Elasticsearch 9 integration tests\n```\n\nEach target builds the appropriate Docker image and runs the corresponding integration test suite.\n\nFor more details, see the [Running Integration Tests](README.md#running-integration-tests) section in the README.\n" }, { "path": "Dockerfile", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n# Simple runtime image that receives a pre-built JAR from the host\nFROM eclipse-temurin:21.0.9_10-jre@sha256:b0f6befb3f2af49704998c4425cb6313c1da505648a8e78cee731531996f735d\nLABEL org.opencontainers.image.authors=\"The Jaeger Authors \"\n\n# Build argument to specify the variant type\n# Supported values: cassandra, elasticsearch7, elasticsearch8, elasticsearch9\nARG VARIANT=elasticsearch9\n\nENV APP_HOME=/app/\nENV VARIANT_TYPE=${VARIANT}\n\n# The JAR is provided by the GHA runner into the artifact-target folder\nCOPY artifact-target/jaeger-spark-dependencies*.jar $APP_HOME/app.jar\n\nWORKDIR $APP_HOME\n\nCOPY entrypoint.sh /\n\nRUN chgrp root /etc/passwd && chmod g+rw /etc/passwd\nUSER 185\n\nENTRYPOINT [\"/entrypoint.sh\"]\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright {yyyy} {name of copyright owner}\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "Makefile", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n.PHONY: e2e-cassandra e2e-es7 e2e-es8 e2e-es9 help\n\nhelp:\n\t@echo \"Available targets:\"\n\t@echo \" e2e-cassandra - Run Cassandra integration tests\"\n\t@echo \" e2e-es7 - Run Elasticsearch 7 integration tests\"\n\t@echo \" e2e-es8 - Run Elasticsearch 8 integration tests\"\n\t@echo \" e2e-es9 - Run Elasticsearch 9 integration tests\"\n\ne2e-cassandra:\n\t@echo \"Building Docker image for Cassandra variant...\"\n\tdocker build \\\n\t --build-arg VARIANT=cassandra \\\n\t -t ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:test-cassandra \\\n\t .\n\t@echo \"Running Cassandra integration tests...\"\n\tSPARK_DEPENDENCIES_JOB_TAG=test-cassandra \\\n\t./mvnw --batch-mode clean test -am -pl jaeger-spark-dependencies-cassandra\n\ne2e-es7:\n\t@echo \"Building Docker image for ES7 variant...\"\n\tdocker build \\\n\t --build-arg VARIANT=elasticsearch7 \\\n\t --build-arg ELASTICSEARCH_SPARK_VERSION=7.17.10 \\\n\t -t ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:test-es7 \\\n\t .\n\t@echo \"Running ES7 integration tests...\"\n\tSPARK_DEPENDENCIES_JOB_TAG=test-es7 \\\n\tELASTICSEARCH_VERSION=7.3.0 \\\n\t./mvnw --batch-mode clean test -am \\\n\t -pl jaeger-spark-dependencies-elasticsearch \\\n\t -Dversion.elasticsearch.spark=7.17.10\n\ne2e-es8:\n\t@echo \"Building Docker image for ES8 variant...\"\n\tdocker build \\\n\t --build-arg VARIANT=elasticsearch8 \\\n\t --build-arg ELASTICSEARCH_SPARK_VERSION=8.13.4 \\\n\t -t ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:test-es8 \\\n\t .\n\t@echo \"Running ES8 integration tests...\"\n\tSPARK_DEPENDENCIES_JOB_TAG=test-es8 \\\n\tELASTICSEARCH_VERSION=8.3.1 \\\n\t./mvnw --batch-mode clean test -am \\\n\t -pl jaeger-spark-dependencies-elasticsearch \\\n\t -Dversion.elasticsearch.spark=8.13.4\n\ne2e-es9:\n\t@echo \"Building Docker image for ES9 variant (unified/mega-jar)...\"\n\tdocker build \\\n\t --build-arg VARIANT=unified \\\n\t --build-arg ELASTICSEARCH_SPARK_VERSION=9.1.3 \\\n\t -t ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:test-es9 \\\n\t .\n\t@echo \"Running ES9 integration tests...\"\n\tSPARK_DEPENDENCIES_JOB_TAG=test-es9 \\\n\tELASTICSEARCH_VERSION=9.1.3 \\\n\t./mvnw --batch-mode clean test -am \\\n\t -pl jaeger-spark-dependencies-elasticsearch \\\n\t -Dversion.elasticsearch.spark=9.1.3\n" }, { "path": "README.md", "content": "[![Latest image](https://ghcr-badge.egpl.dev/jaegertracing/spark-dependencies/spark-dependencies/latest_tag?trim=major&label=latest)](https://github.com/jaegertracing/spark-dependencies/pkgs/container/spark-dependencies%2Fspark-dependencies)\n\n# Jaeger Spark dependencies\nThis is a Spark job that collects spans from storage, analyze links between services,\nand stores them for later presentation in the UI. Note that it is needed for the production deployment.\n`all-in-one` distribution does not need this job.\n\nThis job parses all traces on a given day, based on UTC. By default, it processes the current day,\nbut other days can be explicitly specified.\n\n## Quick-start\nSpark job can be run as docker container and also as java executable:\n\n### Container Image Variants\n\nStarting with version 0.6.x, Docker images are published with variant-specific tags. **Each variant automatically uses the appropriate storage backend, so the `STORAGE` environment variable is no longer needed.**\n\nThe images are named `ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:{VERSION}-{VARIANT}`:\n\n- **`VERSION-cassandra`**: For Cassandra storage (uses CassandraDependenciesJob directly)\n- **`VERSION-elasticsearch7`**: For Elasticsearch 7.12-7.16 (uses ElasticsearchDependenciesJob with ES connector 7.17.29)\n- **`VERSION-elasticsearch8`**: For Elasticsearch 7.17+ and 8.x (uses ElasticsearchDependenciesJob with ES connector 8.13.4)\n- **`VERSION-elasticsearch9`**: For Elasticsearch 9.x (uses ElasticsearchDependenciesJob with ES connector 9.1.3) - also tagged as `:latest`\n- **`VERSION-opensearch`**: For OpenSearch 2.x and 3.x (uses OpenSearchDependenciesJob with OpenSearch Java client)\n\nExample for Cassandra:\n```bash\n$ docker run \\\n --env CASSANDRA_CONTACT_POINTS=host1,host2 \\\n ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:v0.5.3-cassandra\n```\n\nExample for Elasticsearch 8.x:\n```bash\n$ docker run \\\n --env ES_NODES=http://elasticsearch:9200 \\\n ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:v0.5.3-elasticsearch8\n```\n\nExample for OpenSearch:\n```bash\n$ docker run \\\n --env OS_NODES=http://opensearch:9200 \\\n ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:v0.5.3-opensearch\n```\n\n#### Advanced Configuration\n\nUse `--env JAVA_OPTS` to pass additional Java options such as memory settings, SSL trust store, or other JVM properties:\n\n```bash\n# Example: Configure SSL trust store\n$ docker run \\\n --env ES_NODES=https://elasticsearch:9200 \\\n --env JAVA_OPTS=\"-Djavax.net.ssl.trustStore=/path/to/truststore -Djavax.net.ssl.trustStorePassword=changeit\" \\\n ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:v0.5.3-elasticsearch8\n\n# Example: Increase JVM heap size\n$ docker run \\\n --env OS_NODES=http://opensearch:9200 \\\n --env JAVA_OPTS=\"-Xmx2g -Xms1g\" \\\n ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:v0.5.3-opensearch\n```\n\nUse `--env LOG4J_STATUS_LOGGER_LEVEL` to control Log4j2 internal status messages (defaults to `OFF`):\n\n```bash\n# Example: Enable Log4j2 debug logging for troubleshooting\n$ docker run \\\n --env OS_NODES=http://opensearch:9200 \\\n --env LOG4J_STATUS_LOGGER_LEVEL=DEBUG \\\n ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:v0.5.3-opensearch\n```\n\nNote: the latest versions are hosted on `ghcr.io`, not on Docker Hub.\n\nAs jar file:\n```bash\nSTORAGE=cassandra java -jar jaeger-spark-dependencies.jar\n```\n\n## Usage\nBy default, this job parses all traces since midnight UTC. You can parse traces for a different day\nvia an argument in YYYY-mm-dd format, like 2016-07-16 or specify the date via an env property.\n\n```bash\n# ex to run the job to process yesterday's traces on OS/X\n$ STORAGE=cassandra java -jar jaeger-spark-dependencies.jar `date -uv-1d +%F`\n# or on Linux\n$ STORAGE=cassandra java -jar jaeger-spark-dependencies.jar `date -u -d '1 day ago' +%F`\n```\n\n### Configuration\n`jaeger-spark-dependencies` applies configuration parameters through environment variables.\n\nThe following variables are common to all storage layers:\n\n* `SPARK_MASTER`: Spark master to submit the job to; Defaults to `local[*]`\n* `DATE`: Date in YYYY-mm-dd format. Denotes a day for which dependency links will be created.\n* `PEER_SERVICE_TAG`: Tag name used to identify peer service in spans. Defaults to `peer.service`\n* `JAVA_OPTS`: Additional Java options to pass to the JVM. Use this to configure memory, SSL properties, or other JVM settings. Example: `JAVA_OPTS=\"-Xmx2g -Djavax.net.ssl.trustStore=/path/to/truststore\"`. Note: The required `--add-opens` flags for Spark on Java 21+ are already included in the container image.\n* `LOG4J_STATUS_LOGGER_LEVEL`: Log4j2 StatusLogger level. Defaults to `OFF` to suppress internal Log4j2 status messages. Set to `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`, or `FATAL` if you need to debug logging configuration issues.\n\n### Cassandra\nCassandra is used when `STORAGE=cassandra`.\n\n* `CASSANDRA_KEYSPACE`: The keyspace to use. Defaults to \"jaeger_v1_dc1\".\n* `CASSANDRA_CONTACT_POINTS`: Comma separated list of hosts / ip addresses part of Cassandra cluster.\n Defaults to localhost\n* `CASSANDRA_LOCAL_DC`: The local DC to connect to (other nodes will be ignored)\n* `CASSANDRA_USERNAME` and `CASSANDRA_PASSWORD`: Cassandra authentication. Will throw an exception\n on startup if authentication fails\n* `CASSANDRA_USE_SSL`: Requires `javax.net.ssl.trustStore` and `javax.net.ssl.trustStorePassword`,\n Defaults to false.\n* `CASSANDRA_CLIENT_AUTH_ENABLED`: If set enables client authentication on SSL connections.\n Requires `javax.net.ssl.keyStore` and `javax.net.ssl.keyStorePassword`, defaults to false.\n\nExample usage:\n\n```bash\n$ STORAGE=cassandra CASSANDRA_CONTACT_POINTS=localhost:9042 java -jar jaeger-spark-dependencies.jar\n```\n### Elasticsearch\nElasticsearch is used when `STORAGE=elasticsearch`.\n\n**Important**: Use the appropriate Docker image variant for your Elasticsearch version:\n- ES 7.12-7.16: Use `:VERSION-elasticsearch7` tag\n- ES 7.17-8.x: Use `:VERSION-elasticsearch8` tag \n- ES 9.x: Use `:VERSION-elasticsearch9` tag (or `:latest`)\n\n#### Configuration\n\n* `ES_NODES`: A comma separated list of elasticsearch hosts advertising http. Defaults to\n 127.0.0.1. Add port section if not listening on port 9200. Only one of these hosts\n needs to be available to fetch the remaining nodes in the cluster. It is\n recommended to set this to all the master nodes of the cluster. Use url format for\n SSL. For example, \"https://yourhost:8888\"\n* `ES_NODES_WAN_ONLY`: Set to true to only use the values set in ES_NODES, for example if your\n elasticsearch cluster is in Docker. If you're using a cloudprovider\n such as AWS Elasticsearch, set this to true. Defaults to false\n* `ES_USERNAME` and `ES_PASSWORD`: Elasticsearch basic authentication. Use when X-Pack security\n (formerly Shield) is in place. By default no username or password is provided to elasticsearch.\n* `ES_CLIENT_NODE_ONLY`: Set to true to disable elasticsearch cluster nodes.discovery and enable nodes.client.only.\n If your elasticsearch cluster's data nodes only listen on loopback ip, set this to true.\n Defaults to false.\n* `ES_INDEX_PREFIX`: index prefix of Jaeger indices. By default unset.\n* `ES_INDEX_DATE_SEPARATOR`: index date separator of Jaeger indices. The default value is `-`.\n For example `.` will find index \"jaeger-span-2020.11.25\".\n* `ES_TIME_RANGE`: How far in the past the job should look to for spans, the maximum and default is `24h`.\n Any value accepted by [date-math](https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#date-math) can be used here, but the anchor is always `now`.\n* `ES_USE_ALIASES`: Set to true to use index alias names to read from and write to.\n Usually required when using rollover indices.\n\nExample usage:\n\n```bash\n$ STORAGE=elasticsearch ES_NODES=http://localhost:9200 java -jar jaeger-spark-dependencies.jar\n```\n\n### OpenSearch\nOpenSearch is used when `STORAGE=opensearch`.\n\n**Important**: Use the `:VERSION-opensearch` Docker image variant.\n\n#### Configuration\n\n* `OS_NODES`: A comma separated list of OpenSearch hosts advertising http. Defaults to\n 127.0.0.1. Add port section if not listening on port 9200. Only one of these hosts\n needs to be available to fetch the remaining nodes in the cluster. It is\n recommended to set this to all the master nodes of the cluster. Use url format for\n SSL. For example, \"https://yourhost:8888\"\n* `OS_NODES_WAN_ONLY`: Set to true to only use the values set in OS_NODES, for example if your\n OpenSearch cluster is in Docker. If you're using a cloudprovider\n such as AWS OpenSearch, set this to true. Defaults to false.\n* `OS_USERNAME` and `OS_PASSWORD`: OpenSearch basic authentication. By default no username or password is provided.\n* `OS_INDEX_PREFIX`: index prefix of Jaeger indices. By default unset.\n* `OS_INDEX_DATE_SEPARATOR`: index date separator of Jaeger indices. The default value is `-`.\n For example `.` will find index \"jaeger-span-2020.11.25\".\n* `OS_TIME_RANGE`: How far in the past the job should look to for spans, the maximum and default is `24h`.\n Any value accepted by [date-math](https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#date-math) can be used here, but the anchor is always `now`.\n\nExample usage:\n\n```bash\n$ docker run \\\n --env OS_NODES=http://opensearch:9200 \\\n ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:v0.5.3-opensearch\n```\n\n## Design\n\nAt a high-level, this job does the following:\n\n* read lots of spans from a time period\n* group them by traceId\n* construct a graph using parent-child relationships expressed in span references\n* for each edge `(parent span, child span)` output `(parent service, child service, count)`\n* write the results to the database (e.g. `dependencies_v2` table in [Cassandra](https://github.com/jaegertracing/jaeger/blob/12e44faabf10cdd866391b78933eec5d6ac50fa9/plugin/storage/cassandra/schema/v004.cql.tmpl#L186))\n\n## Building locally\nTo build the job locally and run tests:\n```bash\n./mvnw clean install # if failed add SPARK_LOCAL_IP=127.0.0.1\n```\nTo run the unified jar (includes all):\n```bash\nSTORAGE=cassandra java -jar jaeger-spark-dependencies/target/jaeger-spark-dependencies-0.0.1-SNAPSHOT.jar\n#or\nSTORAGE=elasticsearch ES_NODES=http://localhost:9200 java -jar jaeger-spark-dependencies/target/jaeger-spark-dependencies-0.0.1-SNAPSHOT.jar\n#or\nSTORAGE=opensearch OS_NODES=http://localhost:9200 java -jar jaeger-spark-dependencies/target/jaeger-spark-dependencies-0.0.1-SNAPSHOT.jar\n```\n\nTo run storage-specific jars directly (without STORAGE variable):\n```bash\n# Cassandra\njava -jar jaeger-spark-dependencies-cassandra/target/jaeger-spark-dependencies-cassandra-0.0.1-SNAPSHOT.jar\n# Elasticsearch\nES_NODES=http://localhost:9200 java -jar jaeger-spark-dependencies-elasticsearch/target/jaeger-spark-dependencies-elasticsearch-0.0.1-SNAPSHOT.jar\n# OpenSearch\nOS_NODES=http://localhost:9200 java -jar jaeger-spark-dependencies-opensearch/target/jaeger-spark-dependencies-opensearch-0.0.1-SNAPSHOT.jar\n```\n\nTo build Docker image:\n\n**Note:** The Dockerfile now requires a pre-built JAR. First build the JAR using Maven, then build the Docker image.\n\nFor Cassandra:\n```bash\n./mvnw clean package --batch-mode -Dlicense.skip=true -DskipTests -pl jaeger-spark-dependencies-cassandra -am\nmkdir -p artifact-target\ncp jaeger-spark-dependencies-cassandra/target/jaeger-spark-dependencies-cassandra-0.0.1-SNAPSHOT.jar artifact-target/\ndocker build --build-arg VARIANT=cassandra -t jaegertracing/spark-dependencies:cassandra .\n```\n\nFor Elasticsearch 9:\n```bash\n./mvnw clean package --batch-mode -Dlicense.skip=true -DskipTests -Dversion.elasticsearch.spark=9.1.3 -pl jaeger-spark-dependencies-elasticsearch -am\nmkdir -p artifact-target\ncp jaeger-spark-dependencies-elasticsearch/target/jaeger-spark-dependencies-elasticsearch-0.0.1-SNAPSHOT.jar artifact-target/\ndocker build --build-arg VARIANT=elasticsearch9 -t jaegertracing/spark-dependencies:elasticsearch9 .\n```\n\nIn tests it's possible to specify version of Jaeger images by env variable `JAEGER_VERSION`\nor system property `jaeger.version`. By default tests are using latest images.\n\n## Running Integration Tests\n\nThe integration tests validate the Spark dependencies job against different storage backends:\n- Cassandra 4.x\n- Elasticsearch 7\n- Elasticsearch 8\n- Elasticsearch 9\n\n### Prerequisites\n\nBefore running integration tests, ensure you have the following installed:\n\n- **Java 21** (Temurin distribution recommended)\n- **Docker** (for building images and running testcontainers)\n- **Maven** (included via `./mvnw` wrapper)\n\n### Quick Start\n\nUse the following make targets to run integration tests:\n\n```bash\nmake e2e-cassandra # Run Cassandra integration tests\nmake e2e-es7 # Run Elasticsearch 7 integration tests\nmake e2e-es8 # Run Elasticsearch 8 integration tests\nmake e2e-es9 # Run Elasticsearch 9 integration tests\n```\n\n### What Each Target Does\n\nEach test suite performs two steps:\n1. Builds a Docker image with the appropriate storage variant\n2. Runs tests using testcontainers against that variant\n\n### Environment Variables\n\nThe following environment variables are used in integration tests:\n\n- `SPARK_DEPENDENCIES_JOB_TAG`: Specifies the Docker image tag to use in tests (e.g., `test-cassandra`, `test-es7`, `test-es8`, `test-es9`)\n- `ELASTICSEARCH_VERSION`: Specifies the Elasticsearch version for testcontainers to use\n- `JAEGER_VERSION`: (Optional) Specifies the version of Jaeger images to use in tests. Defaults to latest.\n\nYou can also set this as a system property:\n```bash\n./mvnw test -Djaeger.version=2.14.0\n```\n\n### Troubleshooting\n\n#### Docker Permission Issues\nIf you encounter Docker permission issues, ensure your user is in the `docker` group:\n```bash\nsudo usermod -aG docker $USER\n```\nThen log out and log back in.\n\n#### Testcontainers Issues\nIf testcontainers fail to start, ensure:\n1. Docker is running and accessible\n2. The Ryuk image is pulled: `docker pull testcontainersofficial/ryuk:latest`\n3. You have sufficient disk space for Docker images\n\n#### Build Failures\nIf you encounter build failures:\n1. Ensure you have Java 21 installed\n2. Clean the Maven cache: `./mvnw clean`\n3. Try running with the `-U` flag to force update dependencies: `./mvnw -U clean install`\n\n#### Port Conflicts\nIf tests fail due to port conflicts, ensure no other services are running on the ports used by testcontainers (typically ephemeral ports, but sometimes standard ports like 9042 for Cassandra or 9200 for Elasticsearch).\n\n## CI/CD Pipeline\n\nThe project uses a unified CI/CD pipeline (`.github/workflows/ci-cd.yml`) that implements a **Host-Build Matrix Pattern**:\n\n1. **Setup & Dependency Download** - Downloads all Maven dependencies once and warms the cache for subsequent jobs\n2. **Build JARs** - Builds storage-specific JARs on the GitHub runner (parallel for all variants)\n3. **E2E Tests** - Tests each variant using Docker containers with pre-built JARs\n4. **Publish** - Publishes multi-arch Docker images (linux/amd64, linux/arm64) to GitHub Container Registry\n\nThe pipeline supports four variants:\n- `cassandra` - For Cassandra storage\n- `elasticsearch7` - For Elasticsearch 7.12-7.16 (ES connector 7.17.29)\n- `elasticsearch8` - For Elasticsearch 7.17+ and 8.x (ES connector 8.13.4)\n- `elasticsearch9` - For Elasticsearch 9.x (ES connector 9.1.3)\n\nThis approach eliminates Maven downloads inside Docker builds and parallelizes builds across all storage variants.\n\n## License\n\n[Apache 2.0 License](./LICENSE).\n" }, { "path": "RELEASES.md", "content": "# Release process\n\n1. Create a new GitHub release with a new tag\n2. Use \"generate release notes\" button\n3. Once the release is published a `publish release` workflow will build and push container images\n" }, { "path": "entrypoint.sh", "content": "#!/bin/sh\n#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n\n# Taken from https://github.com/radanalyticsio/openshift-spark/blob/2.4/modules/common/added/scripts/entrypoint#L50\n# OpenShift passes random UID and spark requires it to be present in /etc/passwd\npatch_uid() {\n # Check whether there is a passwd entry for the container UID\n myuid=$(id -u)\n mygid=$(id -g)\n uidentry=$(getent passwd $myuid)\n\n # If there is no passwd entry for the container UID, attempt to create one\n if [ -z \"$uidentry\" ] ; then\n if [ -w /etc/passwd ] ; then\n echo \"$myuid:x:$myuid:$mygid:anonymous uid:${PWD}:/bin/false\" >> /etc/passwd\n else\n echo \"Container ENTRYPOINT failed to add passwd entry for anonymous UID\"\n fi\n fi\n}\n\npatch_uid\n\n# Use the single JAR name\nJAR_PATH=\"$APP_HOME/app.jar\"\n\n# Determine main class based on VARIANT_TYPE\nif [ \"$VARIANT_TYPE\" = \"cassandra\" ]; then\n MAIN_CLASS=\"io.jaegertracing.spark.dependencies.cassandra.CassandraDependenciesJob\"\nelif [ -n \"$VARIANT_TYPE\" ] && [ \"${VARIANT_TYPE#elasticsearch}\" != \"$VARIANT_TYPE\" ]; then\n # VARIANT_TYPE starts with \"elasticsearch\"\n MAIN_CLASS=\"io.jaegertracing.spark.dependencies.elastic.ElasticsearchDependenciesJob\"\nelif [ \"$VARIANT_TYPE\" = \"opensearch\" ]; then\n MAIN_CLASS=\"io.jaegertracing.spark.dependencies.opensearch.OpenSearchDependenciesJob\"\nelse\n # Fallback to unified JAR (for backward compatibility or local builds)\n MAIN_CLASS=\"io.jaegertracing.spark.dependencies.DependenciesSparkJob\"\nfi\n\n# Set default Log4j2 StatusLogger level if not already set\n# This suppresses Log4j2 StatusLogger errors triggered by OpenSearch's programmatic logging configuration\n# Users can override this by setting the LOG4J_STATUS_LOGGER_LEVEL environment variable\nLOG4J_STATUS_LOGGER_LEVEL=\"${LOG4J_STATUS_LOGGER_LEVEL:-OFF}\"\n\n# Required Java module options for Spark to work with Java 21+\n# These --add-opens flags are necessary for Spark to access internal Java APIs\nSPARK_JAVA_OPTS=\"--add-opens=java.base/java.lang=ALL-UNNAMED \\\n--add-opens=java.base/java.lang.invoke=ALL-UNNAMED \\\n--add-opens=java.base/java.lang.reflect=ALL-UNNAMED \\\n--add-opens=java.base/java.io=ALL-UNNAMED \\\n--add-opens=java.base/java.net=ALL-UNNAMED \\\n--add-opens=java.base/java.nio=ALL-UNNAMED \\\n--add-opens=java.base/java.util=ALL-UNNAMED \\\n--add-opens=java.base/java.util.concurrent=ALL-UNNAMED \\\n--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \\\n--add-opens=java.base/sun.nio.ch=ALL-UNNAMED \\\n--add-opens=java.base/sun.nio.cs=ALL-UNNAMED \\\n--add-opens=java.base/sun.security.action=ALL-UNNAMED \\\n--add-opens=java.base/sun.util.calendar=ALL-UNNAMED \\\n-Djdk.reflect.useDirectMethodHandle=false\"\n\n# Execute the job with the determined main class\n# SPARK_JAVA_OPTS come first (required for Spark), then JAVA_OPTS (user customizations), then Log4j config\nexec java ${SPARK_JAVA_OPTS} ${JAVA_OPTS} -Dorg.apache.logging.log4j.simplelog.StatusLogger.level=${LOG4J_STATUS_LOGGER_LEVEL} -cp \"$JAR_PATH\" \"$MAIN_CLASS\" \"$@\"\n" }, { "path": "header.txt", "content": "Copyright (c) The Jaeger Authors\nSPDX-License-Identifier: Apache-2.0\n" }, { "path": "jaeger-spark-dependencies/pom.xml", "content": "\n\n\n 4.0.0\n \n io.jaegertracing.dependencies\n jaeger-spark-dependencies-parent\n 0.0.1-SNAPSHOT\n \n\n jaeger-spark-dependencies\n\n \n \n ${project.groupId}\n jaeger-spark-dependencies-cassandra\n \n \n ${project.groupId}\n jaeger-spark-dependencies-elasticsearch\n \n \n\n \n \n \n maven-shade-plugin\n ${version.maven-shade-plugin}\n \n \n package\n \n shade\n \n \n \n \n \n reference.conf\n \n \n io.jaegertracing.spark.dependencies.DependenciesSparkJob\n \n \n \n true\n \n \n \n org.apache.hadoop:hadoop-common\n \n **\n \n \n \n \n com.datastax.oss:*\n \n **\n \n \n \n log4j:log4j\n \n org/apache/log4j/spi/LoggingEvent.class\n \n \n \n org.apache.logging.log4j:log4j-*\n \n **\n \n \n \n io.netty:*\n \n **\n \n \n \n org.slf4j:*\n \n **\n \n \n \n org.scala-lang:*\n \n **\n \n \n \n org.apache.spark:*\n \n **\n \n \n \n org.lz4:*\n \n **\n \n \n \n \n org.elasticsearch:elasticsearch-spark-*\n \n **\n \n \n \n \n commons-httpclient:commons-httpclient\n \n **\n \n \n \n \n xerces:xercesImpl\n \n **\n \n \n \n \n com.squareup.okhttp3:*\n \n **\n \n \n \n \n com.squareup.okio:*\n \n **\n \n \n \n \n *:*\n \n META-INF/*.SF\n META-INF/*.DSA\n META-INF/*.RSA\n \n \n \n false\n \n \n \n \n \n \n\n" }, { "path": "jaeger-spark-dependencies/src/main/java/io/jaegertracing/spark/dependencies/DependenciesSparkJob.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies;\n\nimport io.jaegertracing.spark.dependencies.cassandra.CassandraDependenciesJob;\nimport io.jaegertracing.spark.dependencies.elastic.ElasticsearchDependenciesJob;\nimport java.io.UnsupportedEncodingException;\nimport java.time.LocalDate;\n\npublic final class DependenciesSparkJob {\n\n public static void main(String[] args) throws UnsupportedEncodingException {\n String storage = System.getenv(\"STORAGE\");\n if (storage == null) {\n throw new IllegalArgumentException(\"Missing environmental variable STORAGE\");\n }\n\n LocalDate date = LocalDate.now();\n if (args.length == 1) {\n date = parseZonedDateTime(args[0]);\n } else if (System.getenv(\"DATE\") != null) {\n date = parseZonedDateTime(System.getenv(\"DATE\"));\n }\n\n run(storage, date);\n }\n\n private static void run(String storage, LocalDate localDate) throws UnsupportedEncodingException {\n String peerServiceTag = System.getenv(\"PEER_SERVICE_TAG\");\n if (peerServiceTag == null){\n peerServiceTag = \"peer.service\";\n }\n String jarPath = Utils.pathToUberJar(DependenciesSparkJob.class);\n if (\"elasticsearch\".equalsIgnoreCase(storage)) {\n ElasticsearchDependenciesJob.builder()\n .jars(jarPath)\n .day(localDate)\n .build()\n .run(peerServiceTag);\n } else if (\"cassandra\".equalsIgnoreCase(storage)) {\n CassandraDependenciesJob.builder()\n .jars(jarPath)\n .day(localDate)\n .build()\n .run(peerServiceTag);\n } else {\n throw new IllegalArgumentException(\"Unsupported storage: \" + storage);\n }\n }\n\n static LocalDate parseZonedDateTime(String date) {\n return LocalDate.parse(date);\n }\n}\n" }, { "path": "jaeger-spark-dependencies/src/main/resources/log4j.properties", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n# Set everything to be logged to the console\nlog4j.rootCategory=WARN, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlog4j.logger.org.spark-project.jetty=WARN\nlog4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO\nlog4j.logger.io.jaegertracing.spark=INFO\n\n# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support\nlog4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL\nlog4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR\n" }, { "path": "jaeger-spark-dependencies-cassandra/pom.xml", "content": "\n\n\n 4.0.0\n \n io.jaegertracing.dependencies\n jaeger-spark-dependencies-parent\n 0.0.1-SNAPSHOT\n \n\n jaeger-spark-dependencies-cassandra\n\n \n 3.4.1\n \n\n \n \n ${project.groupId}\n jaeger-spark-dependencies-common\n \n\n \n org.apache.spark\n spark-core_${version.scala.binary}\n \n \n com.fasterxml.jackson.core\n jackson-annotations\n \n \n \n\n \n com.datastax.spark\n spark-cassandra-connector_${version.scala.binary}\n ${spark-cassandra-connector.version}\n \n \n \n org.apache.commons\n commons-compress\n 1.26.0\n \n \n org.eclipse.jetty\n jetty-xml\n 10.0.26\n \n\n \n ${project.groupId}\n jaeger-spark-dependencies-test\n test\n \n \n org.testcontainers\n testcontainers\n test\n \n \n org.testcontainers\n cassandra\n test\n \n \n com.google.guava\n guava\n \n \n commons-lang\n commons-lang\n 2.6\n test\n \n \n\n \n \n \n maven-shade-plugin\n ${version.maven-shade-plugin}\n \n \n package\n \n shade\n \n \n \n \n reference.conf\n \n \n io.jaegertracing.spark.dependencies.cassandra.CassandraDependenciesJob\n \n \n true\n \n \n org.apache.hadoop:hadoop-common\n \n **\n \n \n \n com.datastax.oss:*\n \n **\n \n \n \n com.datastax.spark:*\n \n **\n \n \n \n log4j:log4j\n \n org/apache/log4j/spi/LoggingEvent.class\n \n \n \n org.apache.logging.log4j:log4j-*\n \n **\n \n \n \n io.netty:*\n \n **\n \n \n \n org.slf4j:*\n \n **\n \n \n \n org.scala-lang:*\n \n **\n \n \n \n org.apache.spark:*\n \n **\n \n \n \n org.lz4:*\n \n **\n \n \n \n xerces:xercesImpl\n \n **\n \n \n \n \n com.squareup.okhttp3:*\n \n **\n \n \n \n \n com.squareup.okio:*\n \n **\n \n \n \n *:*\n \n META-INF/*.SF\n META-INF/*.DSA\n META-INF/*.RSA\n \n \n \n false\n \n \n \n \n \n \n\n" }, { "path": "jaeger-spark-dependencies-cassandra/src/main/java/io/jaegertracing/spark/dependencies/cassandra/CassandraDependenciesJob.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * Copyright 2016-2017 The OpenZipkin Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.cassandra;\n\nimport static com.datastax.spark.connector.japi.CassandraJavaUtil.javaFunctions;\nimport static com.datastax.spark.connector.japi.CassandraJavaUtil.mapRowTo;\nimport static com.datastax.spark.connector.japi.CassandraJavaUtil.mapToRow;\n\nimport com.google.common.base.Joiner;\nimport com.google.common.net.HostAndPort;\nimport io.jaegertracing.spark.dependencies.DependenciesSparkHelper;\nimport io.jaegertracing.spark.dependencies.Utils;\nimport io.jaegertracing.spark.dependencies.model.Dependency;\nimport io.jaegertracing.spark.dependencies.model.Span;\nimport java.io.Serializable;\nimport java.text.SimpleDateFormat;\nimport java.time.LocalDate;\nimport java.time.Period;\nimport java.time.ZoneOffset;\nimport java.time.ZonedDateTime;\nimport java.time.temporal.ChronoUnit;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.LinkedHashMap;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TimeZone;\nimport org.apache.spark.SparkConf;\nimport org.apache.spark.api.java.JavaPairRDD;\nimport org.apache.spark.api.java.JavaSparkContext;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport scala.Tuple2;\n\n/**\n * @author OpenZipkin authors\n * @author Pavol Loffay\n */\npublic final class CassandraDependenciesJob {\n private static final Logger log = LoggerFactory.getLogger(CassandraDependenciesJob.class);\n\n public static Builder builder() {\n return new Builder();\n }\n\n public static final class Builder {\n String keyspace = Utils.getEnv(\"CASSANDRA_KEYSPACE\", \"jaeger_v1_dc1\");\n String contactPoints = Utils.getEnv(\"CASSANDRA_CONTACT_POINTS\", \"localhost\");\n String localDc = Utils.getEnv(\"CASSANDRA_LOCAL_DC\", null);\n // local[*] master lets us run & test the job locally without setting a Spark cluster\n String sparkMaster = Utils.getEnv(\"SPARK_MASTER\", \"local[*]\");\n String username = Utils.getEnv(\"CASSANDRA_USERNAME\", \"\");\n String password = Utils.getEnv(\"CASSANDRA_PASSWORD\", \"\");\n // needed when not in local mode\n String[] jars;\n\n // By default the job only works on traces whose first timestamp is today\n ZonedDateTime day = ZonedDateTime.of(LocalDate.now().atStartOfDay(), ZoneOffset.UTC);\n\n final Map sparkProperties = new LinkedHashMap<>();\n\n Builder() {\n sparkProperties.put(\"spark.ui.enabled\", \"false\");\n sparkProperties.put(\"spark.cassandra.connection.ssl.enabled\",\n Utils.getEnv(\"CASSANDRA_USE_SSL\", \"false\"));\n sparkProperties.put(\"spark.cassandra.connection.ssl.trustStore.password\",\n System.getProperty(\"javax.net.ssl.trustStorePassword\", \"\"));\n sparkProperties.put(\"spark.cassandra.connection.ssl.trustStore.path\",\n System.getProperty(\"javax.net.ssl.trustStore\", \"\"));\n sparkProperties.put(\"spark.cassandra.connection.ssl.clientAuth.enabled\", \n \t\t Utils.getEnv(\"CASSANDRA_CLIENT_AUTH_ENABLED\", \"false\"));\n sparkProperties.put(\"spark.cassandra.connection.ssl.keyStore.path\", \n \t\t System.getProperty(\"javax.net.ssl.keyStore\", \"\"));\n sparkProperties.put(\"spark.cassandra.connection.ssl.keyStore.password\", \n \t\t System.getProperty(\"javax.net.ssl.keyStorePassword\", \"\"));\n }\n\n /** When set, this indicates which jars to distribute to the cluster. */\n public Builder jars(String... jars) {\n this.jars = jars;\n return this;\n }\n\n /** Keyspace to store dependency rowsToLinks. Defaults to \"jaeger_v1_test\" */\n public Builder keyspace(String keyspace) {\n Utils.checkNoTNull(\"keyspace\", keyspace);\n this.keyspace = keyspace;\n return this;\n }\n\n /** Cassandra username. */\n public Builder username(String username) {\n Utils.checkNoTNull(\"username\", username);\n this.username = username;\n return this;\n }\n\n /** Cassandra username. */\n public Builder password(String password) {\n Utils.checkNoTNull(\"password\", password);\n this.password = password;\n return this;\n }\n\n /** Day to process dependencies for. Defaults to today. */\n public Builder day(LocalDate day) {\n this.day = day.atStartOfDay(ZoneOffset.UTC);\n return this;\n }\n\n /** Comma separated list of hosts / IPs part of Cassandra cluster. Defaults to localhost */\n public Builder contactPoints(String contactPoints) {\n this.contactPoints = contactPoints;\n return this;\n }\n\n /** The local DC to connect to (other nodes will be ignored) */\n public Builder localDc(String localDc) {\n this.localDc = localDc;\n return this;\n }\n\n public CassandraDependenciesJob build() {\n return new CassandraDependenciesJob(this);\n }\n\n }\n\n private final String keyspace;\n private final ZonedDateTime day;\n private final SparkConf conf;\n\n CassandraDependenciesJob(Builder builder) {\n this.keyspace = builder.keyspace;\n this.day = builder.day;\n SimpleDateFormat df = new SimpleDateFormat(\"yyyy-MM-dd\");\n df.setTimeZone(TimeZone.getTimeZone(\"UTC\"));\n this.conf = new SparkConf(true)\n .setMaster(builder.sparkMaster)\n .setAppName(getClass().getName());\n conf.set(\"spark.cassandra.connection.host\", parseHosts(builder.contactPoints));\n conf.set(\"spark.cassandra.connection.port\", parsePort(builder.contactPoints));\n conf.set(\"spark.cassandra.auth.username\", builder.username);\n conf.set(\"spark.cassandra.auth.password\", builder.password);\n if (builder.localDc != null) {\n conf.set(\"connection.local_dc\", builder.localDc);\n }\n if (builder.jars != null) {\n conf.setJars(builder.jars);\n }\n for (Map.Entry entry : builder.sparkProperties.entrySet()) {\n conf.set(entry.getKey(), entry.getValue());\n }\n }\n\n public void run(String peerServiceTag) {\n long microsLower = day.toInstant().toEpochMilli() * 1000;\n long microsUpper = day.plus(Period.ofDays(1)).toInstant().toEpochMilli() * 1000 - 1;\n\n log.info(\"Running Dependencies job for {}: {} ≤ Span.timestamp {}\", day, microsLower, microsUpper);\n JavaSparkContext sc = new JavaSparkContext(conf);\n try {\n JavaPairRDD> traces = javaFunctions(sc)\n .cassandraTable(keyspace, \"traces\", mapRowTo(CassandraSpan.class))\n .where(\"start_time < ? AND start_time > ?\", microsUpper, microsLower)\n .mapToPair(span -> new Tuple2<>(span.getTraceId(), span))\n .mapValues(span -> (Span) span)\n .groupByKey();\n\n List dependencyLinks = DependenciesSparkHelper.derive(traces,peerServiceTag);\n store(sc, dependencyLinks);\n log.info(\"Done, {} dependency objects created\", dependencyLinks.size());\n } finally {\n sc.stop();\n }\n }\n\n private void store(JavaSparkContext sc, List links) {\n String table = dependenciesTable(sc);\n log.info(\"Storing dependencies into {}\", table);\n if (\"dependencies_v2\".equals(table)) {\n CassandraDependenciesV2 dependencies = new CassandraDependenciesV2(links, day);\n javaFunctions(sc.parallelize(Collections.singletonList(dependencies)))\n .writerBuilder(keyspace, table, mapToRow(CassandraDependenciesV2.class))\n .saveToCassandra();\n } else {\n CassandraDependencies dependencies = new CassandraDependencies(links, day);\n javaFunctions(sc.parallelize(Collections.singletonList(dependencies)))\n .writerBuilder(keyspace, table, mapToRow(CassandraDependencies.class))\n .saveToCassandra();\n }\n }\n\n static String parseHosts(String contactPoints) {\n List result = new LinkedList<>();\n for (String contactPoint : contactPoints.split(\",\")) {\n HostAndPort parsed = HostAndPort.fromString(contactPoint);\n result.add(parsed.getHost());\n }\n return Joiner.on(',').join(result);\n }\n\n /** Returns the consistent port across all contact points or 9042 */\n static String parsePort(String contactPoints) {\n Set ports = new HashSet<>();\n for (String contactPoint: contactPoints.split(\",\")) {\n HostAndPort parsed = HostAndPort.fromString(contactPoint);\n ports.add(parsed.getPortOrDefault(9042));\n }\n return ports.size() == 1 ? String.valueOf(ports.iterator().next()) : \"9042\";\n }\n\n private String dependenciesTable(JavaSparkContext sc) {\n try {\n javaFunctions(sc)\n .cassandraTable(keyspace, \"dependencies_v2\")\n .limit(1L).collect();\n } catch (Exception ex) {\n return \"dependencies\";\n }\n return \"dependencies_v2\";\n }\n\n /**\n * DTO object used to store dependencies to Cassandra, see {@link com.datastax.spark.connector.mapper.JavaBeanColumnMapper}\n */\n public final static class CassandraDependencies implements Serializable {\n private static final long serialVersionUID = 0L;\n\n private List dependencies;\n private ZonedDateTime zonedDateTime;\n\n public CassandraDependencies(List dependencies, ZonedDateTime ts) {\n this.dependencies = dependencies;\n this.zonedDateTime = ts;\n }\n\n public List getDependencies() {\n return dependencies;\n }\n\n public Long getTs() {\n return zonedDateTime.toInstant().toEpochMilli();\n }\n\n public Long getTsIndex() {\n return zonedDateTime.toInstant().toEpochMilli();\n }\n }\n\n /**\n * DTO object used to store dependencies to Cassandra, see {@link com.datastax.spark.connector.mapper.JavaBeanColumnMapper}\n */\n public final static class CassandraDependenciesV2 implements Serializable {\n private static final long serialVersionUID = 0L;\n\n private List dependencies;\n private ZonedDateTime zonedDateTime;\n\n public CassandraDependenciesV2(List dependencies, ZonedDateTime ts) {\n this.dependencies = dependencies;\n this.zonedDateTime = ts;\n }\n\n public List getDependencies() {\n return dependencies;\n }\n\n public Long getTs() {\n return zonedDateTime.toInstant().toEpochMilli();\n }\n\n public Long getTsBucket() {\n return zonedDateTime.toInstant().truncatedTo(ChronoUnit.DAYS).toEpochMilli();\n }\n }\n\n /**\n * Entry point for running CassandraDependenciesJob directly.\n * This is used when the Docker image variant is cassandra-specific.\n */\n public static void main(String[] args) throws java.io.UnsupportedEncodingException {\n LocalDate date = LocalDate.now();\n if (args.length == 1) {\n date = LocalDate.parse(args[0]);\n } else if (System.getenv(\"DATE\") != null) {\n date = LocalDate.parse(System.getenv(\"DATE\"));\n }\n\n String peerServiceTag = System.getenv(\"PEER_SERVICE_TAG\");\n if (peerServiceTag == null) {\n peerServiceTag = \"peer.service\";\n }\n\n String jarPath = Utils.pathToUberJar(CassandraDependenciesJob.class);\n CassandraDependenciesJob.builder()\n .jars(jarPath)\n .day(date)\n .build()\n .run(peerServiceTag);\n }\n}\n\n" }, { "path": "jaeger-spark-dependencies-cassandra/src/main/java/io/jaegertracing/spark/dependencies/cassandra/CassandraSpan.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.cassandra;\n\nimport io.jaegertracing.spark.dependencies.model.Reference;\nimport io.jaegertracing.spark.dependencies.model.Span;\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * Jaeger > 1.5 does not store parentId. All references are stored in references table.\n * This class is used to maintain compatibility with older Jaeger deployments.\n *\n * @author Pavol Loffay\n */\npublic class CassandraSpan extends Span {\n\n private Long parentId;\n\n public Long getParentId() {\n return parentId;\n }\n\n public void setParentId(Long parentId) {\n this.parentId = parentId;\n }\n\n @Override\n public List getRefs() {\n ArrayList references = new ArrayList<>(super.getRefs());\n Reference legacyParent = new Reference();\n legacyParent.setSpanId(parentId);\n references.add(legacyParent);\n return references;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-cassandra/src/main/resources/log4j2.component.properties", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n# Disable Log4j status logger console output\nlog4j2.StatusLogger.level = OFF\n" }, { "path": "jaeger-spark-dependencies-cassandra/src/main/resources/log4j2.properties", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n# Set root logger level to WARN and use console appender\nrootLogger.level = WARN\nrootLogger.appenderRef.console.ref = console\n\n# Console appender configuration\nappender.console.type = Console\nappender.console.name = console\nappender.console.target = SYSTEM_ERR\nappender.console.layout.type = PatternLayout\nappender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlogger.jetty.name = org.spark-project.jetty\nlogger.jetty.level = WARN\n\nlogger.jettyLifecycle.name = org.spark-project.jetty.util.component.AbstractLifeCycle\nlogger.jettyLifecycle.level = ERROR\n\nlogger.sparkReplTyper.name = org.apache.spark.repl.SparkIMain$exprTyper\nlogger.sparkReplTyper.level = INFO\n\nlogger.sparkReplInterpreter.name = org.apache.spark.repl.SparkILoop$SparkILoopInterpreter\nlogger.sparkReplInterpreter.level = INFO\n\nlogger.jaegertracing.name = io.jaegertracing.spark\nlogger.jaegertracing.level = INFO\n\n# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support\nlogger.hiveMetastore.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler\nlogger.hiveMetastore.level = FATAL\n\nlogger.hiveFunctionRegistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry\nlogger.hiveFunctionRegistry.level = ERROR\n" }, { "path": "jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/CassandraDependenciesDockerJobTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.cassandra;\n\nimport io.jaegertracing.spark.dependencies.LogToConsolePrinter;\nimport org.testcontainers.containers.GenericContainer;\nimport org.testcontainers.utility.DockerImageName;\n\nimport java.util.concurrent.TimeUnit;\n\nimport static org.awaitility.Awaitility.await;\n\npublic class CassandraDependenciesDockerJobTest extends CassandraDependenciesJobTest {\n private static String dependenciesJobTag() {\n String tag = System.getenv(\"SPARK_DEPENDENCIES_JOB_IMAGE_TAG\");\n if (tag == null || tag.isEmpty()) {\n throw new IllegalStateException(\n \"SPARK_DEPENDENCIES_JOB_IMAGE_TAG environment variable is required but not set. \" +\n \"This variable must be set to ensure tests use the locally built Docker image.\");\n }\n return tag.trim();\n }\n\n @Override\n protected void deriveDependencies() {\n System.out.println(\"::group::🚧 🚧 🚧 CassandraDependenciesDockerJob logs\");\n try (GenericContainer sparkDependenciesJob = new GenericContainer<>(\n DockerImageName.parse(\"ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:\" + dependenciesJobTag()))\n .withNetwork(network)\n .withLogConsumer(new LogToConsolePrinter(\"[spark-dependencies] \"))\n .withEnv(\"CASSANDRA_KEYSPACE\", \"jaeger_v1_dc1\")\n .withEnv(\"CASSANDRA_CONTACT_POINTS\", \"cassandra\") // This should be an address within the docker network\n .withEnv(\"CASSANDRA_LOCAL_DC\", cassandra.getLocalDatacenter())\n .withEnv(\"CASSANDRA_USERNAME\", cassandra.getUsername())\n .withEnv(\"CASSANDRA_PASSWORD\", cassandra.getPassword())\n .dependsOn(cassandra, jaegerCassandraSchema);) {\n sparkDependenciesJob.start();\n await(\"spark-dependencies-job execution\")\n .atMost(3, TimeUnit.MINUTES)\n .until(() -> !sparkDependenciesJob.isRunning());\n } finally {\n System.out.println(\"::endgroup::\");\n }\n }\n}\n" }, { "path": "jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/CassandraDependenciesJobTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.cassandra;\n\nimport static org.awaitility.Awaitility.await;\n\nimport com.datastax.oss.driver.api.core.CqlSession;\nimport io.jaegertracing.spark.dependencies.LogToConsolePrinter;\nimport io.jaegertracing.spark.dependencies.test.DependenciesTest;\nimport java.time.LocalDate;\nimport java.util.Collections;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.concurrent.TimeUnit;\nimport org.junit.After;\nimport org.junit.AfterClass;\nimport org.junit.BeforeClass;\nimport org.testcontainers.cassandra.CassandraContainer;\nimport org.testcontainers.containers.GenericContainer;\nimport org.testcontainers.containers.Network;\nimport org.testcontainers.containers.wait.strategy.HttpWaitStrategy;\n\n/**\n * @author Pavol Loffay\n */\npublic class CassandraDependenciesJobTest extends DependenciesTest {\n\n protected static Network network;\n protected static CassandraContainer cassandra;\n protected static GenericContainer jaegerAll;\n protected static GenericContainer jaegerCassandraSchema;\n private static int cassandraPort;\n\n @BeforeClass\n public static void beforeClass() {\n System.out.println(\"=== Starting CassandraDependenciesJobTest setup ===\");\n\n network = Network.newNetwork();\n System.out.println(\"Created network: \" + network.getId());\n\n System.out.println(\"Starting Cassandra container (cassandra:4.1)...\");\n cassandra = new CassandraContainer(\"cassandra:4.1\")\n .withNetwork(network)\n .withNetworkAliases(\"cassandra\")\n .withExposedPorts(9042);\n cassandra.start();\n cassandraPort = cassandra.getMappedPort(9042);\n System.out.println(\"Cassandra started. Mapped port: \" + cassandraPort);\n\n System.out.println(\"Starting Jaeger Cassandra schema container (jaegertracing/jaeger-cassandra-schema:\"\n + jaegerVersion() + \")...\");\n jaegerCassandraSchema = new GenericContainer<>(\"jaegertracing/jaeger-cassandra-schema:\" + jaegerVersion())\n .withLogConsumer(new LogToConsolePrinter(\"[jaeger-cassandra-schema] \"))\n .withNetwork(network);\n jaegerCassandraSchema.start();\n System.out.println(\"Jaeger Cassandra schema container started, waiting for schema creation...\");\n /**\n * Wait until schema is created\n */\n await().until(() -> !jaegerCassandraSchema.isRunning());\n System.out.println(\"Jaeger Cassandra schema creation completed\");\n\n System.out.println(\"Starting Jaeger v2 unified container (jaegertracing/jaeger:\" + jaegerVersion() + \")...\");\n jaegerAll = new GenericContainer<>(\"jaegertracing/jaeger:\" + jaegerVersion())\n .withNetwork(network)\n .withClasspathResourceMapping(\"jaeger-v2-config-cassandra.yaml\", \"/etc/jaeger/config.yaml\",\n org.testcontainers.containers.BindMode.READ_ONLY)\n .withCommand(\"--config\", \"/etc/jaeger/config.yaml\")\n .waitingFor(new BoundPortHttpWaitStrategy(16687)\n .forStatusCodeMatching(statusCode -> statusCode >= 200 && statusCode < 300))\n .withExposedPorts(16687, 16686, 4317, 4318, 14268, 9411);\n jaegerAll.start();\n System.out.println(\"Jaeger v2 container started\");\n\n queryUrl = String.format(\"http://127.0.0.1:%d\", jaegerAll.getMappedPort(16686));\n collectorUrl = String.format(\"http://127.0.0.1:%d\", jaegerAll.getMappedPort(4317));\n\n System.out.println(\"=== Container setup complete ===\");\n System.out.println(\"Query URL: \" + queryUrl);\n System.out.println(\"Collector URL: \" + collectorUrl);\n System.out.println(\"Health check port: \" + jaegerAll.getMappedPort(16687));\n }\n\n @AfterClass\n public static void afterClass() {\n Optional.of(cassandra).ifPresent(GenericContainer::close);\n Optional.of(jaegerAll).ifPresent(GenericContainer::close);\n Optional.of(jaegerCassandraSchema).ifPresent(GenericContainer::close);\n }\n\n @After\n public void after() {\n try (CqlSession session = CqlSession.builder()\n .addContactPoint(cassandra.getContactPoint())\n .withLocalDatacenter(cassandra.getLocalDatacenter())\n .build()) {\n session.execute(\"TRUNCATE jaeger_v1_dc1.traces\");\n session.execute(String.format(\"TRUNCATE jaeger_v1_dc1.%s\", dependenciesTable(session)));\n }\n }\n\n private String dependenciesTable(CqlSession session) {\n try {\n session.execute(\"SELECT ts from jaeger_v1_dc1.dependencies_v2 limit 1;\");\n } catch (Exception ex) {\n return \"dependencies\";\n }\n return \"dependencies_v2\";\n }\n\n @Override\n protected void deriveDependencies() {\n System.out.println(\"::group::🚧 🚧 🚧 CassandraDependenciesJob logs\");\n try {\n CassandraDependenciesJob.builder()\n .contactPoints(\"localhost:\" + cassandraPort)\n .day(LocalDate.now())\n .keyspace(\"jaeger_v1_dc1\")\n .username(cassandra.getUsername())\n .password(cassandra.getPassword())\n .build()\n .run(\"peer.service\");\n } finally {\n System.out.println(\"::endgroup::\");\n }\n }\n\n @Override\n protected void waitBetweenTraces() throws InterruptedException {\n // TODO otherwise it sometimes fails\n TimeUnit.SECONDS.sleep(1);\n }\n\n public static class BoundPortHttpWaitStrategy extends HttpWaitStrategy {\n private final int port;\n\n public BoundPortHttpWaitStrategy(int port) {\n this.port = port;\n }\n\n @Override\n protected Set getLivenessCheckPorts() {\n int mapptedPort = this.waitStrategyTarget.getMappedPort(port);\n return Collections.singleton(mapptedPort);\n }\n }\n}\n" }, { "path": "jaeger-spark-dependencies-cassandra/src/test/java/io/jaegertracing/spark/dependencies/cassandra/JaegerTestDriverContainer.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.cassandra;\n\nimport java.net.ConnectException;\nimport java.time.Duration;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.TimeUnit;\nimport okhttp3.OkHttpClient;\nimport okhttp3.Request;\nimport okhttp3.Response;\nimport org.rnorth.ducttape.unreliables.Unreliables;\nimport org.testcontainers.containers.ContainerLaunchException;\nimport org.testcontainers.containers.GenericContainer;\nimport org.testcontainers.containers.traits.LinkableContainer;\n\n/**\n * @author Pavol Loffay\n */\npublic class JaegerTestDriverContainer extends GenericContainer\n implements LinkableContainer {\n protected final OkHttpClient okHttpClient = new OkHttpClient.Builder().build();\n protected final Duration waitUntilReady;\n\n public JaegerTestDriverContainer(String dockerImageName) {\n this(dockerImageName, Duration.ofMinutes(1));\n }\n\n public JaegerTestDriverContainer(String dockerImageName, Duration waitUntilReady) {\n super(dockerImageName);\n this.waitUntilReady = waitUntilReady;\n }\n\n @Override\n protected void waitUntilContainerStarted() {\n String statusUrl = String.format(\"http://localhost:%d/\", this.getMappedPort(8080));\n Unreliables.retryUntilTrue((int)waitUntilReady.toMillis(), TimeUnit.MILLISECONDS, containerStartedCondition(statusUrl));\n }\n\n protected Callable containerStartedCondition(String statusUrl) {\n return () -> {\n if (!isRunning()) {\n throw new ContainerLaunchException(\"Container failed to start\");\n }\n\n Request request = new Request.Builder()\n .url(statusUrl)\n .head()\n .build();\n try (Response response = okHttpClient.newCall(request).execute()) {\n return response.code() == 200;\n } catch (ConnectException ex) {\n return false;\n }\n };\n }\n}\n" }, { "path": "jaeger-spark-dependencies-cassandra/src/test/resources/jaeger-v2-config-cassandra.yaml", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\nservice:\n extensions: [jaeger_storage, jaeger_query, healthcheckv2]\n pipelines:\n traces:\n receivers: [otlp, jaeger, zipkin]\n processors: [filter/jaeger, batch]\n exporters: [jaeger_storage_exporter]\n telemetry:\n resource:\n service.name: jaeger-backend\n metrics:\n level: detailed\n readers:\n - pull:\n exporter:\n prometheus:\n host: 0.0.0.0\n port: 8888\n logs:\n level: info\n traces:\n level: none\n\nextensions:\n healthcheckv2:\n use_v2: true\n http:\n endpoint: \"0.0.0.0:16687\"\n status:\n enabled: true\n path: \"/\"\n\n jaeger_query:\n storage:\n traces: some_storage\n\n jaeger_storage:\n backends:\n some_storage:\n cassandra:\n schema:\n keyspace: \"jaeger_v1_dc1\"\n connection:\n servers: [\"cassandra:9042\"]\n tls:\n insecure: true\n\nreceivers:\n otlp:\n protocols:\n grpc:\n endpoint: \"0.0.0.0:4317\"\n http:\n endpoint: \"0.0.0.0:4318\"\n\n jaeger:\n protocols:\n grpc:\n thrift_binary:\n thrift_compact:\n thrift_http:\n endpoint: \"0.0.0.0:14268\"\n\n zipkin:\n endpoint: \"0.0.0.0:9411\"\n\nprocessors:\n filter/jaeger:\n error_mode: ignore\n traces:\n span:\n - 'resource.attributes[\"service.name\"] == \"jaeger\"'\n \n batch:\n\nexporters:\n jaeger_storage_exporter:\n trace_storage: some_storage\n" }, { "path": "jaeger-spark-dependencies-cassandra/src/test/resources/log4j.properties", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n# Set everything to be logged to the console\nlog4j.rootCategory=WARN, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlog4j.logger.org.spark-project.jetty=WARN\nlog4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO\n\n# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support\nlog4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL\nlog4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR\n\nlog4j.logger.io.jaegertracing=WARN\nlog4j.logger.org.testcontainers=INFO\nlog4j.logger.org.apache.http=WARN\nlog4j.logger.com.github.dockerjava=WARN\nlog4j.logger.org.zeroturnaround.exec=WARN\n" }, { "path": "jaeger-spark-dependencies-common/pom.xml", "content": "\n\n\n 4.0.0\n \n jaeger-spark-dependencies-parent\n io.jaegertracing.dependencies\n 0.0.1-SNAPSHOT\n \n\n jaeger-spark-dependencies-common\n\n \n \n io.opentracing\n opentracing-api\n ${version.io.opentracing}\n \n \n\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/DependenciesSparkHelper.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies;\n\nimport io.jaegertracing.spark.dependencies.model.Dependency;\nimport io.jaegertracing.spark.dependencies.model.Span;\nimport java.util.List;\nimport org.apache.spark.api.java.JavaPairRDD;\nimport scala.Tuple2;\n\n/**\n * @author Pavol Loffay\n */\npublic class DependenciesSparkHelper {\n private DependenciesSparkHelper() {}\n\n /**\n * Derives dependency links based on supplied spans (e.g. multiple traces). If there is a link A->B\n * in multiple traces it will return just one {@link Dependency} link with a correct {@link Dependency#callCount}.\n * Note that RDDs are grouped on traceId so if a span contains multiple references from different traces\n * the job does not produce correct result.\n *\n * @param traceIdSpans {@link org.apache.spark.api.java.JavaRDD} with trace id and a collection of\n * spans with that traceId.\n * @return Aggregated dependency links for all traces.\n */\n public static List derive(JavaPairRDD> traceIdSpans,String peerServiceTag) {\n return traceIdSpans.flatMapValues(new SpansToDependencyLinks(peerServiceTag))\n .values()\n .mapToPair(dependency -> new Tuple2<>(new Tuple2<>(dependency.getParent(), dependency.getChild()), dependency))\n .reduceByKey((v1, v2) -> new Dependency(v1.getParent(), v1.getChild(), v1.getCallCount() + v2.getCallCount()))\n .values()\n .collect();\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/SpansToDependencyLinks.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies;\n\nimport io.jaegertracing.spark.dependencies.model.Dependency;\nimport io.jaegertracing.spark.dependencies.model.KeyValue;\nimport io.jaegertracing.spark.dependencies.model.Reference;\nimport io.jaegertracing.spark.dependencies.model.Span;\nimport io.opentracing.tag.Tags;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.Set;\n\nimport org.apache.spark.api.java.function.FlatMapFunction;\n\n/**\n * @author Pavol Loffay\n */\npublic class SpansToDependencyLinks implements FlatMapFunction, Dependency>{\n\n /**\n * Derives dependency links based on supplied spans.\n *\n * @param trace trace\n * @return collection of dependency links, note that it contains duplicates\n * @throws Exception\n */\n\n public String peerServiceTag = \"\";\n\n public SpansToDependencyLinks(String peerServiceTag){\n this.peerServiceTag = peerServiceTag;\n }\n\n @Override\n public java.util.Iterator call(Iterable trace) {\n Set uniqueSpans = new LinkedHashSet<>();\n for (Span span : trace) {\n uniqueSpans.add(span);\n }\n\n Map> spanMap = new LinkedHashMap<>();\n Map> spanChildrenMap = new LinkedHashMap<>();\n for (Span span : uniqueSpans) {\n // Map of children\n for (Reference ref: span.getRefs()){\n Set children = spanChildrenMap.get(ref.getSpanId());\n if (children == null){\n children = new LinkedHashSet<>();\n spanChildrenMap.put(ref.getSpanId(), children);\n }\n children.add(span);\n }\n // Map of parents\n Set sharedSpans = spanMap.get(span.getSpanId());\n if (sharedSpans == null) {\n sharedSpans = new LinkedHashSet<>();\n spanMap.put(span.getSpanId(), sharedSpans);\n }\n sharedSpans.add(span);\n }\n\n // Let's start with zipkin shared spans\n List result = sharedSpanDependencies(spanMap);\n\n for (Span span : uniqueSpans) {\n if (span.getRefs() == null || span.getRefs().isEmpty() ||\n span.getProcess() == null || span.getProcess().getServiceName() == null) {\n continue;\n }\n\n // if the current span is shared and not a client span we skip it\n // because the link from this span to parent should be from client span\n if (spanMap.get(span.getSpanId()).size() > 1 && !isClientSpan(span)) {\n continue;\n }\n\n for (Reference reference: span.getRefs()) {\n Set parents = spanMap.get(reference.getSpanId());\n if (parents != null) {\n if (parents.size() > 1) {\n serverSpan(parents)\n .ifPresent(parent ->\n result.add(new Dependency(parent.getProcess().getServiceName(), span.getProcess().getServiceName()))\n );\n } else {\n // this is jaeger span or zipkin native (not shared!)\n Span parent = parents.iterator().next();\n if (parent.getProcess() == null || parent.getProcess().getServiceName() == null) {\n continue;\n }\n result.add(new Dependency(parent.getProcess().getServiceName(), span.getProcess().getServiceName()));\n }\n }\n }\n // We are on a leaf so we try to add a dependency for calls to components that calls remote components not instrumented\n if (spanChildrenMap.get(span.getSpanId()) == null ){\n String targetName = span.getTag(peerServiceTag);\n if (targetName != null) {\n result.add(new Dependency(span.getProcess().getServiceName(), targetName));\n }\n }\n }\n return result.iterator();\n }\n\n static Optional serverSpan(Set sharedSpans) {\n for (Span span: sharedSpans) {\n if (isServerSpan(span)) {\n return Optional.of(span);\n }\n }\n\n return Optional.empty();\n }\n\n static boolean isClientSpan(Span span) {\n return Tags.SPAN_KIND_CLIENT.equals(span.getTag(Tags.SPAN_KIND.getKey()));\n }\n\n static boolean isServerSpan(Span span) {\n return Tags.SPAN_KIND_SERVER.equals(span.getTag(Tags.SPAN_KIND.getKey()));\n }\n\n private List sharedSpanDependencies(Map> spanMap) {\n List dependencies = new ArrayList<>();\n // create links between shared spans\n for (Set sharedSpans: spanMap.values()) {\n sharedSpanDependency(sharedSpans)\n .ifPresent(dependencies::add);\n }\n return dependencies;\n }\n\n protected Optional sharedSpanDependency(Set sharedSpans) {\n String clientService = null;\n String serverService = null;\n for (Span span: sharedSpans) {\n for (KeyValue tag: span.getTags()) {\n if (Tags.SPAN_KIND_CLIENT.equals(tag.getValueString()) || Tags.SPAN_KIND_PRODUCER.equals(tag.getValueString())) {\n clientService = span.getProcess().getServiceName();\n } else if (Tags.SPAN_KIND_SERVER.equals(tag.getValueString()) || Tags.SPAN_KIND_CONSUMER.equals(tag.getValueString())) {\n serverService = span.getProcess().getServiceName();\n }\n\n if (clientService != null && serverService != null) {\n return Optional.of(new Dependency(clientService, serverService));\n }\n }\n }\n return Optional.empty();\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/Utils.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies;\n\nimport java.io.UnsupportedEncodingException;\nimport java.net.URL;\nimport java.net.URLDecoder;\n\n/**\n * @author Pavol Loffay\n */\npublic class Utils {\n private Utils() {}\n\n public static String getEnv(String key, String defaultValue) {\n String result = System.getenv(key);\n return result != null ? result : defaultValue;\n }\n\n public static void checkNoTNull(String msg, Object object) {\n if (object == null) {\n throw new NullPointerException(String.format(\"%s is null\", msg));\n }\n }\n\n /**\n * Returns the path to the uber jar containing the calling class.\n * This is used to distribute the jar to Spark workers.\n */\n public static String pathToUberJar(Class clazz) throws UnsupportedEncodingException {\n URL jarFile = clazz.getProtectionDomain().getCodeSource().getLocation();\n return URLDecoder.decode(jarFile.getPath(), \"UTF-8\");\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/json/JsonHelper.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.json;\n\nimport com.fasterxml.jackson.databind.DeserializationFeature;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport io.jaegertracing.spark.dependencies.model.KeyValue;\nimport io.jaegertracing.spark.dependencies.model.Reference;\nimport io.jaegertracing.spark.dependencies.model.Span;\n\n/**\n * @author Pavol Loffay\n * @author Danish Siddiqui\n */\npublic class JsonHelper {\n\n private JsonHelper() {\n }\n\n public static ObjectMapper configure(ObjectMapper objectMapper) {\n objectMapper.addMixIn(Span.class, SpanMixin.class);\n objectMapper.addMixIn(KeyValue.class, KeyValueMixin.class);\n objectMapper.addMixIn(Reference.class, ReferenceMixin.class);\n objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);\n return objectMapper;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/json/KeyValueDeserializer.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.json;\n\nimport com.fasterxml.jackson.core.JsonParser;\nimport com.fasterxml.jackson.core.JsonProcessingException;\nimport com.fasterxml.jackson.databind.DeserializationContext;\nimport com.fasterxml.jackson.databind.JsonNode;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.fasterxml.jackson.databind.deser.std.StdDeserializer;\nimport io.jaegertracing.spark.dependencies.model.KeyValue;\nimport java.io.IOException;\n\n/**\n * @author Pavol Loffay\n * @author Danish Siddiqui\n */\npublic class KeyValueDeserializer extends StdDeserializer {\n\n // TODO Spark incorrectly serializes object mapper, therefore reinitializing\n // here\n private ObjectMapper objectMapper = JsonHelper.configure(new ObjectMapper());\n\n public KeyValueDeserializer() {\n super(KeyValue.class);\n }\n\n @Override\n public KeyValue deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException {\n JsonNode node = objectMapper.getFactory().setCodec(objectMapper).getCodec().readTree(jp);\n\n String key = node.get(\"key\").asText();\n String type = node.get(\"type\").asText();\n\n KeyValue keyValue = new KeyValue();\n keyValue.setKey(key);\n keyValue.setValueType(type);\n\n if (\"string\".equalsIgnoreCase(type)) {\n JsonNode valueNode = node.get(\"value\");\n if (valueNode != null) {\n keyValue.setValueString(valueNode.asText());\n }\n } else {\n // TODO: KeyValue model only supports string value for now, other types are\n // ignored\n }\n\n return keyValue;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/json/KeyValueMixin.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.json;\n\nimport com.fasterxml.jackson.databind.annotation.JsonDeserialize;\n\n/**\n * @author Pavol Loffay\n * @author Danish Siddiqui\n */\n@JsonDeserialize(using = KeyValueDeserializer.class)\npublic class KeyValueMixin {\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/json/ReferenceDeserializer.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.json;\n\nimport com.fasterxml.jackson.core.JsonParser;\nimport com.fasterxml.jackson.core.JsonProcessingException;\nimport com.fasterxml.jackson.databind.DeserializationContext;\nimport com.fasterxml.jackson.databind.JsonNode;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.fasterxml.jackson.databind.deser.std.StdDeserializer;\nimport io.jaegertracing.spark.dependencies.model.Reference;\nimport java.io.IOException;\nimport java.math.BigInteger;\n\n/**\n * @author Pavol Loffay\n * @author Danish Siddiqui\n */\npublic class ReferenceDeserializer extends StdDeserializer {\n\n private ObjectMapper objectMapper = JsonHelper.configure(new ObjectMapper());\n\n protected ReferenceDeserializer() {\n super(Reference.class);\n }\n\n @Override\n public Reference deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException {\n JsonNode node = objectMapper.getFactory().setCodec(objectMapper).getCodec().readTree(jp);\n\n String spanIdHex = node.get(\"spanID\").asText();\n\n Reference reference = new Reference();\n reference.setSpanId(new BigInteger(spanIdHex, 16).longValue());\n return reference;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/json/ReferenceMixin.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.json;\n\nimport com.fasterxml.jackson.databind.annotation.JsonDeserialize;\n\n/**\n * @author Pavol Loffay\n * @author Danish Siddiqui\n */\n@JsonDeserialize(using = ReferenceDeserializer.class)\npublic class ReferenceMixin {\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/json/SpanDeserializer.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.json;\n\nimport com.fasterxml.jackson.core.JsonParser;\nimport com.fasterxml.jackson.core.JsonProcessingException;\nimport com.fasterxml.jackson.databind.DeserializationContext;\nimport com.fasterxml.jackson.databind.JsonNode;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.fasterxml.jackson.databind.deser.std.StdDeserializer;\nimport io.jaegertracing.spark.dependencies.model.KeyValue;\nimport io.jaegertracing.spark.dependencies.model.Process;\nimport io.jaegertracing.spark.dependencies.model.Reference;\nimport io.jaegertracing.spark.dependencies.model.Span;\nimport java.io.IOException;\nimport java.math.BigInteger;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.stream.Collectors;\n\n/**\n * @author Pavol Loffay\n * @author Danish Siddiqui\n */\npublic class SpanDeserializer extends StdDeserializer {\n\n // TODO Spark incorrectly serializes object mapper, therefore reinitializing\n // here\n private ObjectMapper objectMapper = JsonHelper.configure(new ObjectMapper());\n\n public SpanDeserializer() {\n super(Span.class);\n }\n\n @Override\n public Span deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException {\n JsonNode node = objectMapper.getFactory().setCodec(objectMapper).getCodec().readTree(jp);\n\n JsonNode spanIdNode = node.get(\"spanID\");\n JsonNode traceIdNode = node.get(\"traceID\");\n JsonNode startTimeNode = node.get(\"startTime\");\n\n if (spanIdNode == null || traceIdNode == null) {\n throw new JsonProcessingException(\"Missing required fields: spanID or traceID\") {\n };\n }\n\n String spanIdHex = spanIdNode.asText();\n String traceIdHex = traceIdNode.asText();\n String startTimeStr = startTimeNode != null ? startTimeNode.asText() : null;\n\n JsonNode processNode = node.get(\"process\");\n Process process = objectMapper.treeToValue(processNode, Process.class);\n\n JsonNode tagsNode = node.get(\"tags\");\n List tags = Arrays.asList(objectMapper.treeToValue(tagsNode, KeyValue[].class));\n\n JsonNode tagFieldsNode = node.get(\"tag\");\n if (tagFieldsNode != null) {\n Map tagFields = objectMapper.treeToValue(tagFieldsNode, Map.class);\n tags = addTagFields(tags, tagFields);\n }\n\n Span span = new Span();\n span.setSpanId(new BigInteger(spanIdHex, 16).longValue());\n span.setTraceId(traceIdHex);\n span.setRefs(deserializeReferences(node));\n span.setStartTime(startTimeStr != null ? Long.parseLong(startTimeStr) : null);\n span.setProcess(process);\n span.setTags(tags);\n return span;\n }\n\n private List addTagFields(List tags, Map tagFields) {\n ArrayList result = new ArrayList<>(tags.size() + tagFields.size());\n result.addAll(tags);\n List collect = tagFields.entrySet().stream().map(stringObjectEntry -> {\n KeyValue kv = new KeyValue();\n kv.setKey(stringObjectEntry.getKey());\n kv.setValueString(stringObjectEntry.getValue().toString());\n return kv;\n }).collect(Collectors.toList());\n result.addAll(collect);\n return result;\n }\n\n private List deserializeReferences(JsonNode node) throws JsonProcessingException {\n List references = new ArrayList<>();\n JsonNode parentSpanID = node.get(\"parentSpanID\");\n if (parentSpanID != null) {\n BigInteger bigInteger = new BigInteger(parentSpanID.asText(), 16);\n Reference reference = new Reference();\n reference.setSpanId(bigInteger.longValue());\n references.add(reference);\n }\n\n JsonNode referencesNode = node.get(\"references\");\n if (!referencesNode.isNull()) {\n Reference[] referencesArr = objectMapper.treeToValue(referencesNode, Reference[].class);\n references.addAll(Arrays.asList(referencesArr));\n }\n\n return references;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/json/SpanMixin.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.json;\n\nimport com.fasterxml.jackson.databind.annotation.JsonDeserialize;\n\n/**\n * @author Pavol Loffay\n * @author Danish Siddiqui\n */\n@JsonDeserialize(using = SpanDeserializer.class)\npublic class SpanMixin {\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/model/Dependency.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.model;\n\nimport java.io.Serializable;\n\n/**\n * @author Pavol Loffay\n */\npublic class Dependency implements Serializable {\n private static final long serialVersionUID = 0L;\n\n private final String parent;\n private final String child;\n private final long callCount;\n\n public Dependency(String parent, String child) {\n this(parent, child, 1);\n }\n\n public Dependency(String parent, String child, long callCount) {\n this.parent = parent;\n this.child = child;\n this.callCount = callCount;\n }\n\n public String getParent() {\n return parent;\n }\n\n public String getChild() {\n return child;\n }\n\n public long getCallCount() {\n return callCount;\n }\n\n @Override\n public boolean equals(Object o) {\n if (this == o) {\n return true;\n }\n if (!(o instanceof Dependency)) {\n return false;\n }\n\n Dependency that = (Dependency) o;\n\n if (!parent.equals(that.parent)) {\n return false;\n }\n return (this.parent.equals(that.parent))\n && (this.child.equals(that.child))\n && this.callCount == that.callCount;\n }\n\n public String getSource() {\n return \"jaeger\";\n }\n\n public void setSource(String source) {\n }\n\n @Override\n public int hashCode() {\n int h = 1;\n h *= 1000003;\n h ^= parent.hashCode();\n h *= 1000003;\n h ^= child.hashCode();\n h *= 1000003;\n h ^= (int) (h ^ ((callCount >>> 32) ^ callCount));\n h *= 1000003;\n return h;\n }\n\n @Override\n public String toString() {\n return \"Dependency{\" +\n \"parent='\" + parent + '\\'' +\n \", child='\" + child + '\\'' +\n \", callCount=\" + callCount +\n '}';\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/model/KeyValue.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.model;\n\nimport java.io.Serializable;\n\n/**\n * @author Pavol Loffay\n */\npublic class KeyValue implements Serializable {\n private static final long serialVersionUID = 0L;\n\n private String key;\n private String valueType;\n\n // TODO there are more types: double, long, binary, not needed at the moment\n private String valueString;\n\n public String getKey() {\n return key;\n }\n\n public void setKey(String key) {\n this.key = key;\n }\n\n public String getValueString() {\n return valueString;\n }\n\n public void setValueString(String valueString) {\n this.valueString = valueString;\n }\n\n public String getValueType() {\n return valueType;\n }\n\n public void setValueType(String valueType) {\n this.valueType = valueType;\n }\n\n @Override\n public boolean equals(Object o) {\n if (this == o) {\n return true;\n }\n if (o == null || getClass() != o.getClass()) {\n return false;\n }\n\n KeyValue keyValue = (KeyValue) o;\n\n if (key != null ? !key.equals(keyValue.key) : keyValue.key != null) {\n return false;\n }\n if (valueType != null ? !valueType.equals(keyValue.valueType) : keyValue.valueType != null) {\n return false;\n }\n return valueString != null ? valueString.equals(keyValue.valueString) : keyValue.valueString == null;\n }\n\n @Override\n public int hashCode() {\n int result = key != null ? key.hashCode() : 0;\n result = 31 * result + (valueType != null ? valueType.hashCode() : 0);\n result = 31 * result + (valueString != null ? valueString.hashCode() : 0);\n return result;\n }\n}" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/model/Process.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.model;\n\nimport java.io.Serializable;\n\n/**\n * @author Pavol Loffay\n */\npublic class Process implements Serializable {\n private static final long serialVersionUID = 0L;\n\n private String serviceName;\n\n public String getServiceName() {\n return serviceName;\n }\n\n public void setServiceName(String serviceName) {\n this.serviceName = serviceName;\n }\n\n @Override\n public boolean equals(Object o) {\n if (this == o) {\n return true;\n }\n if (!(o instanceof Process)) {\n return false;\n }\n\n Process process = (Process) o;\n\n return serviceName != null ? serviceName.equals(process.serviceName)\n : process.serviceName == null;\n }\n\n @Override\n public int hashCode() {\n return serviceName != null ? serviceName.hashCode() : 0;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/model/Reference.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.model;\n\nimport java.io.Serializable;\n\n/**\n * @author Pavol Loffay\n */\npublic class Reference implements Serializable {\n private static final long serialVersionUID = 0L;\n\n private Long spanId;\n\n public Long getSpanId() {\n return spanId;\n }\n\n public void setSpanId(Long spanId) {\n this.spanId = spanId;\n }\n\n @Override\n public boolean equals(Object o) {\n if (this == o) {\n return true;\n }\n if (o == null || getClass() != o.getClass()) {\n return false;\n }\n\n Reference reference = (Reference) o;\n\n return spanId != null ? spanId.equals(reference.spanId) : reference.spanId == null;\n }\n\n @Override\n public int hashCode() {\n return spanId != null ? spanId.hashCode() : 0;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/main/java/io/jaegertracing/spark/dependencies/model/Span.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.model;\n\nimport java.io.Serializable;\nimport java.util.List;\n\n/**\n * @author Pavol Loffay\n */\npublic class Span implements Serializable {\n\n private static final long serialVersionUID = 0L;\n\n private String traceId;\n private Long spanId;\n\n private Long startTime;\n private Process process;\n private List tags;\n private List refs;\n\n public String getTraceId() {\n return traceId;\n }\n\n public void setTraceId(String traceId) {\n this.traceId = traceId;\n }\n\n public Long getSpanId() {\n return spanId;\n }\n\n public void setSpanId(Long spanId) {\n this.spanId = spanId;\n }\n\n public long getStartTime() {\n return startTime;\n }\n\n public void setStartTime(Long startTime) {\n this.startTime = startTime;\n }\n\n public Process getProcess() {\n return process;\n }\n\n public void setProcess(Process process) {\n this.process = process;\n }\n\n public List getTags() {\n return tags;\n }\n\n public String getTag(String key){\n for (KeyValue kv : tags){\n if (kv.getKey().equals(key)){\n return kv.getValueString();\n }\n }\n return null;\n }\n\n public void setTags(List tags) {\n this.tags = tags;\n }\n\n public List getRefs() {\n return refs;\n }\n\n public void setRefs(List refs) {\n this.refs = refs;\n }\n\n @Override\n public boolean equals(Object o) {\n if (this == o) {\n return true;\n }\n if (o == null || getClass() != o.getClass()) {\n return false;\n }\n\n Span span = (Span) o;\n\n if (traceId != null ? !traceId.equals(span.traceId) : span.traceId != null) {\n return false;\n }\n if (spanId != null ? !spanId.equals(span.spanId) : span.spanId != null) {\n return false;\n }\n if (startTime != null ? !startTime.equals(span.startTime) : span.startTime != null) {\n return false;\n }\n if (process != null ? !process.equals(span.process) : span.process != null) {\n return false;\n }\n if (tags != null ? !tags.equals(span.tags) : span.tags != null) {\n return false;\n }\n return refs != null ? refs.equals(span.refs) : span.refs == null;\n }\n\n @Override\n public int hashCode() {\n int result = traceId != null ? traceId.hashCode() : 0;\n result = 31 * result + (spanId != null ? spanId.hashCode() : 0);\n result = 31 * result + (startTime != null ? startTime.hashCode() : 0);\n result = 31 * result + (process != null ? process.hashCode() : 0);\n result = 31 * result + (tags != null ? tags.hashCode() : 0);\n result = 31 * result + (refs != null ? refs.hashCode() : 0);\n return result;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/test/java/io/jaegertracing/spark/dependencies/SpansToDependencyLinksTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertFalse;\nimport static org.junit.Assert.assertTrue;\n\nimport io.jaegertracing.spark.dependencies.model.Dependency;\nimport io.jaegertracing.spark.dependencies.model.KeyValue;\nimport io.jaegertracing.spark.dependencies.model.Process;\nimport io.jaegertracing.spark.dependencies.model.Span;\nimport io.opentracing.tag.Tags;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.Set;\nimport org.junit.Test;\n\npublic class SpansToDependencyLinksTest {\n\n @Test\n public void shouldReturnDependencyWithClientAndServerSpans() {\n SpansToDependencyLinks spansToDependencyLinks = new SpansToDependencyLinks(\"\");\n Set sharedSpans = new HashSet<>();\n sharedSpans.add(createSpan(\"clientName\", Tags.SPAN_KIND_CLIENT));\n sharedSpans.add(createSpan(\"serverName\", Tags.SPAN_KIND_SERVER));\n Optional result = spansToDependencyLinks.sharedSpanDependency(sharedSpans);\n assertTrue(result.isPresent());\n assertEquals(new Dependency(\"clientName\", \"serverName\"), result.get());\n }\n\n @Test\n public void shouldReturnDependencyWithConsumerAndProducer() {\n SpansToDependencyLinks spansToDependencyLinks = new SpansToDependencyLinks(\"\");\n Set sharedSpans = new HashSet<>();\n sharedSpans.add(createSpan(\"consumerName\", Tags.SPAN_KIND_CONSUMER));\n sharedSpans.add(createSpan(\"producerName\", Tags.SPAN_KIND_PRODUCER));\n Optional result = spansToDependencyLinks.sharedSpanDependency(sharedSpans);\n assertTrue(result.isPresent());\n assertEquals(new Dependency(\"producerName\", \"consumerName\"), result.get());\n }\n\n @Test\n public void shouldReturnEmptyDependencyForSpansWithoutSpanKindDefinition() {\n SpansToDependencyLinks spansToDependencyLinks = new SpansToDependencyLinks(\"\");\n Set sharedSpans = new HashSet<>();\n sharedSpans.add(createSpan(\"consumerName\", \"tag\"));\n sharedSpans.add(createSpan(\"producerName\", \"tag\"));\n Optional result = spansToDependencyLinks.sharedSpanDependency(sharedSpans);\n assertFalse(result.isPresent());\n }\n\n private Span createSpan(String serviceName, String tag) {\n List tags = new ArrayList<>();\n KeyValue keyValue = new KeyValue();\n keyValue.setKey(\"span.kind\");\n keyValue.setValueString(tag);\n tags.add(keyValue);\n Span span = new Span();\n Process process = new Process();\n process.setServiceName(serviceName);\n span.setProcess(process);\n span.setTags(tags);\n return span;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-common/src/test/java/io/jaegertracing/spark/dependencies/model/SpanTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.model;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotEquals;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.junit.Test;\n\npublic class SpanTest {\n\n @Test\n public void testEquals() {\n Span span1 = new Span();\n span1.setTraceId(\"trace1\");\n span1.setSpanId(1L);\n span1.setProcess(createProcess(\"service1\"));\n\n Span span2 = new Span();\n span2.setTraceId(\"trace1\");\n span2.setSpanId(1L);\n span2.setProcess(createProcess(\"service1\"));\n\n assertEquals(span1, span2);\n\n // Different service name\n Span span3 = new Span();\n span3.setTraceId(\"trace1\");\n span3.setSpanId(1L);\n span3.setProcess(createProcess(\"service2\"));\n\n assertNotEquals(span1, span3);\n\n // Different tags\n List tags1 = new ArrayList<>();\n KeyValue kv1 = new KeyValue();\n kv1.setKey(\"key\");\n kv1.setValueString(\"value1\");\n tags1.add(kv1);\n span1.setTags(tags1);\n\n List tags2 = new ArrayList<>();\n KeyValue kv2 = new KeyValue();\n kv2.setKey(\"key\");\n kv2.setValueString(\"value1\");\n tags2.add(kv2);\n span2.setTags(tags2);\n\n assertEquals(span1, span2);\n\n tags2.get(0).setValueString(\"value2\");\n assertNotEquals(span1, span2);\n }\n\n private Process createProcess(String serviceName) {\n Process process = new Process();\n process.setServiceName(serviceName);\n return process;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-elasticsearch/pom.xml", "content": "\n\n\n 4.0.0\n \n jaeger-spark-dependencies-parent\n io.jaegertracing.dependencies\n 0.0.1-SNAPSHOT\n \n\n jaeger-spark-dependencies-elasticsearch\n\n \n \n ${project.groupId}\n jaeger-spark-dependencies-common\n \n \n com.fasterxml.jackson.core\n jackson-annotations\n \n \n \n\n \n org.elasticsearch\n elasticsearch-spark-30_${version.scala.binary}\n ${version.elasticsearch.spark}\n \n\n \n ${project.groupId}\n jaeger-spark-dependencies-test\n test\n \n \n org.testcontainers\n testcontainers\n test\n \n \n org.awaitility\n awaitility\n ${version.org.awaitility-awaitility}\n test\n \n \n io.opentelemetry\n opentelemetry-api\n ${version.io.opentelemetry}\n test\n \n \n\n \n \n \n maven-shade-plugin\n ${version.maven-shade-plugin}\n \n \n package\n \n shade\n \n \n \n \n reference.conf\n \n \n io.jaegertracing.spark.dependencies.elastic.ElasticsearchDependenciesJob\n \n \n true\n \n \n org.apache.hadoop:hadoop-common\n \n **\n \n \n \n log4j:log4j\n \n org/apache/log4j/spi/LoggingEvent.class\n \n \n \n org.apache.logging.log4j:log4j-*\n \n **\n \n \n \n io.netty:*\n \n **\n \n \n \n org.slf4j:*\n \n **\n \n \n \n org.scala-lang:*\n \n **\n \n \n \n org.apache.spark:*\n \n **\n \n \n \n org.lz4:*\n \n **\n \n \n \n org.elasticsearch:elasticsearch-spark-*\n \n **\n \n \n \n commons-httpclient:commons-httpclient\n \n **\n \n \n \n xerces:xercesImpl\n \n **\n \n \n \n \n com.squareup.okhttp3:*\n \n **\n \n \n \n \n com.squareup.okio:*\n \n **\n \n \n \n *:*\n \n META-INF/*.SF\n META-INF/*.DSA\n META-INF/*.RSA\n \n \n \n false\n \n \n \n \n \n \n\n" }, { "path": "jaeger-spark-dependencies-elasticsearch/src/main/java/io/jaegertracing/spark/dependencies/elastic/ElasticTupleToSpan.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.elastic;\n\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport io.jaegertracing.spark.dependencies.json.JsonHelper;\nimport io.jaegertracing.spark.dependencies.model.Span;\nimport org.apache.spark.api.java.function.Function;\nimport scala.Tuple2;\n\n/**\n * @author Pavol Loffay\n */\npublic class ElasticTupleToSpan implements Function, Span> {\n\n private ObjectMapper objectMapper = JsonHelper.configure(new ObjectMapper());\n\n @Override\n public Span call(Tuple2 tuple) throws Exception {\n Span span = objectMapper.readValue(tuple._2(), Span.class);\n String originalTraceId = span.getTraceId();\n span.setTraceId(normalizeTraceId(originalTraceId));\n if (span.getTags() != null) {\n span.getTags().sort((o1, o2) -> o1.getKey().compareTo(o2.getKey()));\n }\n if (span.getRefs() != null) {\n span.getRefs().sort((o1, o2) -> o1.getSpanId().compareTo(o2.getSpanId()));\n }\n\n return span;\n }\n\n private String normalizeTraceId(String traceId) {\n if (traceId != null && traceId.length() < 32) {\n return String.format(\"%32s\", traceId).replace(' ', '0');\n }\n return traceId;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-elasticsearch/src/main/java/io/jaegertracing/spark/dependencies/elastic/ElasticsearchDependenciesJob.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * Copyright 2016-2017 The OpenZipkin Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.elastic;\n\nimport com.fasterxml.jackson.core.JsonProcessingException;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport io.jaegertracing.spark.dependencies.DependenciesSparkHelper;\nimport io.jaegertracing.spark.dependencies.Utils;\nimport io.jaegertracing.spark.dependencies.model.Dependency;\nimport io.jaegertracing.spark.dependencies.model.Span;\nimport java.time.LocalDate;\nimport java.time.ZoneOffset;\nimport java.time.ZonedDateTime;\nimport java.time.format.DateTimeFormatter;\nimport java.util.Collections;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.regex.Pattern;\nimport org.apache.spark.SparkConf;\nimport org.apache.spark.api.java.JavaPairRDD;\nimport org.apache.spark.api.java.JavaSparkContext;\nimport org.elasticsearch.hadoop.rest.RestClient;\nimport org.elasticsearch.hadoop.util.EsMajorVersion;\nimport org.elasticsearch.spark.cfg.SparkSettings;\nimport org.elasticsearch.spark.rdd.api.java.JavaEsSpark;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n * @author OpenZipkin authors\n * @author Pavol Loffay\n */\npublic class ElasticsearchDependenciesJob {\n private static final Logger log = LoggerFactory.getLogger(ElasticsearchDependenciesJob.class);\n private static final Pattern PORT_PATTERN = Pattern.compile(\":\\\\d+\");\n\n public static Builder builder() {\n return new Builder();\n }\n\n public static final class Builder {\n\n String hosts = Utils.getEnv(\"ES_NODES\", \"127.0.0.1\");\n String username = Utils.getEnv(\"ES_USERNAME\", null);\n String password = Utils.getEnv(\"ES_PASSWORD\", null);\n Boolean clientNodeOnly = Boolean.parseBoolean(Utils.getEnv(\"ES_CLIENT_NODE_ONLY\", \"false\"));\n Boolean nodesWanOnly = Boolean.parseBoolean(Utils.getEnv(\"ES_NODES_WAN_ONLY\", \"false\"));\n String indexPrefix = Utils.getEnv(\"ES_INDEX_PREFIX\", null);\n String indexDatePattern = datePattern(Utils.getEnv(\"ES_INDEX_DATE_SEPARATOR\", \"-\"));\n String spanRange = Utils.getEnv(\"ES_TIME_RANGE\", \"24h\");\n Boolean useAliases = Boolean.parseBoolean(Utils.getEnv(\"ES_USE_ALIASES\", \"false\"));\n\n final Map sparkProperties = new LinkedHashMap<>();\n\n Builder() {\n sparkProperties.put(\"spark.ui.enabled\", \"false\");\n // don't die if there are no spans\n sparkProperties.put(\"es.index.read.missing.as.empty\", \"true\");\n sparkProperties.put(\"es.net.ssl.keystore.location\",\n getSystemPropertyAsFileResource(\"javax.net.ssl.keyStore\"));\n sparkProperties.put(\"es.net.ssl.keystore.pass\",\n System.getProperty(\"javax.net.ssl.keyStorePassword\", \"\"));\n sparkProperties.put(\"es.net.ssl.truststore.location\",\n getSystemPropertyAsFileResource(\"javax.net.ssl.trustStore\"));\n sparkProperties.put(\"es.net.ssl.truststore.pass\",\n System.getProperty(\"javax.net.ssl.trustStorePassword\", \"\"));\n\n }\n\n // local[*] master lets us run & test the job locally without setting a Spark cluster\n String sparkMaster = Utils.getEnv(\"SPARK_MASTER\", \"local[*]\");\n // needed when not in local mode\n String[] jars;\n\n // By default the job only works on traces whose first timestamp is today\n ZonedDateTime day = ZonedDateTime.of(LocalDate.now().atStartOfDay(), ZoneOffset.UTC);\n\n /** When set, this indicates which jars to distribute to the cluster. */\n public Builder jars(String... jars) {\n this.jars = jars;\n return this;\n }\n\n /** es.nodes separated by ',' */\n public Builder nodes(String hosts) {\n Utils.checkNoTNull(hosts, \"nodes\");\n this.hosts = hosts;\n this.nodesWanOnly = true;\n return this;\n }\n\n /** username used for basic auth. Needed when Shield or X-Pack security is enabled */\n public Builder username(String username) {\n this.username = username;\n return this;\n }\n\n /** password used for basic auth. Needed when Shield or X-Pack security is enabled */\n public Builder password(String password) {\n this.password = password;\n return this;\n }\n\n /** index prefix for Jaeger indices. By default empty */\n public Builder indexPrefix(String indexPrefix) {\n this.indexPrefix = indexPrefix;\n return this;\n }\n\n /** index date pattern for Jaeger indices. By default yyyy-MM-dd */\n public Builder indexDatePattern(String indexDatePattern) {\n this.indexDatePattern = indexDatePattern;\n return this;\n }\n\n /** span range for Jaeger indices. By default 24h */\n public Builder spanRange(String spanRange) {\n this.spanRange = spanRange;\n return this;\n }\n\n /** Day to process dependencies for. Defaults to today. */\n public Builder day(LocalDate day) {\n this.day = day.atStartOfDay(ZoneOffset.UTC);\n return this;\n }\n\n /** Whether the connector is used against an Elasticsearch instance in a cloud/restricted\n * environment over the WAN, such as Amazon Web Services. In this mode, the\n * connector disables discovery and only connects through the declared es.nodes during all operations,\n * including reads and writes. Note that in this mode, performance is highly affected. */\n public Builder nodesWanOnly(boolean wanOnly) {\n this.nodesWanOnly = wanOnly;\n return this;\n }\n\n private static void logIfNoPort(String hosts) {\n if (!PORT_PATTERN.matcher(hosts).find()) {\n log.warn(\"Port is not specified, default port 9200 will be used\");\n }\n }\n\n public ElasticsearchDependenciesJob build() {\n String hosts = System.getenv(\"ES_NODES\");\n String wanOnly = System.getenv(\"ES_NODES_WAN_ONLY\");\n // Optimize user configuration - nodes specified but wan only not\n if (hosts != null && wanOnly == null) {\n this.nodesWanOnly = true;\n }\n logIfNoPort(this.hosts);\n return new ElasticsearchDependenciesJob(this);\n }\n }\n\n private static String getSystemPropertyAsFileResource(String key) {\n String prop = System.getProperty(key, \"\");\n return prop != null && !prop.isEmpty() ? \"file:\" + prop : prop;\n }\n\n private final ZonedDateTime day;\n private final SparkConf conf;\n private final String indexPrefix;\n private final String indexDatePattern;\n private final String spanRange;\n private final Boolean useAliases;\n\n ElasticsearchDependenciesJob(Builder builder) {\n this.day = builder.day;\n this.conf = new SparkConf(true).setMaster(builder.sparkMaster).setAppName(getClass().getName());\n if (builder.jars != null) {\n conf.setJars(builder.jars);\n }\n if (builder.username != null) {\n conf.set(\"es.net.http.auth.user\", builder.username);\n }\n if (builder.password != null) {\n conf.set(\"es.net.http.auth.pass\", builder.password);\n }\n conf.set(\"es.nodes\", builder.hosts);\n if (builder.hosts.indexOf(\"https\") != -1) {\n conf.set(\"es.net.ssl\", \"true\");\n }\n if (builder.nodesWanOnly) {\n conf.set(\"es.nodes.wan.only\", \"true\");\n }\n if (builder.clientNodeOnly) {\n conf.set(\"es.nodes.discovery\", \"0\");\n conf.set(\"es.nodes.client.only\", \"1\");\n }\n for (Map.Entry entry : builder.sparkProperties.entrySet()) {\n conf.set(entry.getKey(), entry.getValue());\n }\n this.indexPrefix = builder.indexPrefix;\n this.indexDatePattern = builder.indexDatePattern;\n this.spanRange = builder.spanRange;\n this.useAliases = builder.useAliases;\n }\n\n /**\n * https://github.com/jaegertracing/jaeger/blob/master/CHANGELOG.md#190-2019-01-21\n */\n private static String prefixBefore19(String prefix) {\n return prefix != null ? String.format(\"%s:\", prefix) : \"\";\n }\n\n private static String prefix(String prefix) {\n return prefix != null ? String.format(\"%s-\", prefix) : \"\";\n }\n\n private static String datePattern(String separator) {\n if (separator.equals(\"\")) {\n return \"yyyyMMdd\";\n }\n // ' is escape character in date format, we should double it here.\n if (separator.contains(\"'\")) {\n separator = separator.replace(\"'\", \"''\");\n }\n return String.format(\"yyyy'%s'MM'%s'dd\", separator, separator);\n }\n\n public void run(String peerServiceTag) {\n\n String[] readIndices;\n String[] writeIndex;\n\n // use alias indices common when using index rollover\n if (this.useAliases) {\n readIndices = new String[]{prefix(indexPrefix) + \"jaeger-span-read\", prefixBefore19(indexPrefix) + \"jaeger-span-read\"};\n writeIndex = new String[] {prefix(indexPrefix) + \"jaeger-dependencies-write\", prefixBefore19(indexPrefix) + \"jaeger-dependencies-write\"};\n }\n else {\n readIndices = indexDate(\"jaeger-span\");\n writeIndex = indexDate(\"jaeger-dependencies\");\n }\n\n run(readIndices, writeIndex, peerServiceTag);\n }\n\n String[] indexDate(String index) {\n String date = day.toLocalDate().format(DateTimeFormatter.ofPattern(indexDatePattern));\n if (indexPrefix != null && indexPrefix.length() > 0) {\n return new String[]{String.format(\"%s%s-%s\", prefix(indexPrefix), index, date), String.format(\"%s%s-%s\", prefixBefore19(indexPrefix), index, date)};\n }\n // if there is no prefix we read and write only to one index\n return new String[]{String.format(\"%s-%s\", index, date)};\n }\n\n void run(String[] spanIndices, String[] depIndices,String peerServiceTag) {\n JavaSparkContext sc = new JavaSparkContext(conf);\n try {\n for (int i = 0; i < spanIndices.length; i++) {\n String spanIndex = spanIndices[i];\n String depIndex = depIndices[i];\n log.info(\"Running Dependencies job for {}, reading from {} index, result storing to {}\", day, spanIndex, depIndex);\n // Send raw query to ES to select only the docs / spans we want to consider for this job\n // This doesn't change the default behavior as the daily indexes only contain up to 24h of data\n String esQuery = String.format(\"{\\\"range\\\": {\\\"startTimeMillis\\\": { \\\"gte\\\": \\\"now-%s\\\" }}}\", spanRange);\n JavaPairRDD> traces = JavaEsSpark.esJsonRDD(sc, spanIndex, esQuery)\n .map(new ElasticTupleToSpan())\n .groupBy(Span::getTraceId);\n List dependencyLinks = DependenciesSparkHelper.derive(traces,peerServiceTag);\n EsMajorVersion esMajorVersion = getEsVersion();\n // Add type for ES < 7\n // WARN log is produced for older ES versions, however it's produced by spark-es library and not ES itself, it cannot be disabled\n // WARN Resource: Detected type name in resource [jaeger-dependencies-2019-08-14/dependencies]. Type names are deprecated and will be removed in a later release.\n if (esMajorVersion.before(EsMajorVersion.V_7_X)) {\n depIndex = depIndex + \"/dependencies\";\n }\n store(sc, dependencyLinks, depIndex);\n log.info(\"Done, {} dependency objects created\", dependencyLinks.size());\n if (dependencyLinks.size() > 0) {\n // we do not derive dependencies for old prefix \"prefix:\" if new prefix \"prefix-\" contains data\n break;\n }\n }\n } finally {\n sc.stop();\n }\n }\n\n private EsMajorVersion getEsVersion() {\n RestClient client = new RestClient(new SparkSettings(conf));\n try {\n return client.mainInfo().getMajorVersion();\n } finally {\n client.close();\n }\n }\n\n private void store(JavaSparkContext javaSparkContext, List dependencyLinks, String resource) {\n if (dependencyLinks.isEmpty()) {\n return;\n }\n\n String json;\n try {\n ObjectMapper objectMapper = new ObjectMapper();\n json = objectMapper.writeValueAsString(new ElasticsearchDependencies(dependencyLinks, day));\n } catch (JsonProcessingException e) {\n throw new IllegalStateException(\"Could not serialize dependencies\", e);\n }\n\n JavaEsSpark.saveJsonToEs(javaSparkContext.parallelize(Collections.singletonList(json)), resource);\n }\n\n /**\n * Helper class used to serialize dependencies to JSON.\n */\n public static final class ElasticsearchDependencies {\n private List dependencies;\n private ZonedDateTime ts;\n\n public ElasticsearchDependencies(List dependencies, ZonedDateTime ts) {\n this.dependencies = dependencies;\n this.ts = ts;\n }\n\n public List getDependencies() {\n return dependencies;\n }\n\n public String getTimestamp() {\n // Jaeger ES dependency storage uses RFC3339Nano for timestamp\n return ts.format(DateTimeFormatter.ofPattern(\"yyyy-MM-dd'T'HH:mm:ssXXX\"));\n }\n }\n\n /**\n * Entry point for running ElasticsearchDependenciesJob directly.\n * This is used when the Docker image variant is elasticsearch-specific.\n */\n public static void main(String[] args) throws java.io.UnsupportedEncodingException {\n LocalDate date = LocalDate.now();\n if (args.length == 1) {\n date = LocalDate.parse(args[0]);\n } else if (System.getenv(\"DATE\") != null) {\n date = LocalDate.parse(System.getenv(\"DATE\"));\n }\n\n String peerServiceTag = System.getenv(\"PEER_SERVICE_TAG\");\n if (peerServiceTag == null) {\n peerServiceTag = \"peer.service\";\n }\n\n String jarPath = Utils.pathToUberJar(ElasticsearchDependenciesJob.class);\n ElasticsearchDependenciesJob.builder()\n .jars(jarPath)\n .day(date)\n .build()\n .run(peerServiceTag);\n }\n}\n" }, { "path": "jaeger-spark-dependencies-elasticsearch/src/main/resources/log4j2.component.properties", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n# Disable Log4j status logger console output\nlog4j2.StatusLogger.level = OFF\n" }, { "path": "jaeger-spark-dependencies-elasticsearch/src/main/resources/log4j2.properties", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n# Set root logger level to WARN and use console appender\nrootLogger.level = WARN\nrootLogger.appenderRef.console.ref = console\n\n# Console appender configuration\nappender.console.type = Console\nappender.console.name = console\nappender.console.target = SYSTEM_ERR\nappender.console.layout.type = PatternLayout\nappender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlogger.jetty.name = org.spark-project.jetty\nlogger.jetty.level = WARN\n\nlogger.jettyLifecycle.name = org.spark-project.jetty.util.component.AbstractLifeCycle\nlogger.jettyLifecycle.level = ERROR\n\nlogger.sparkReplTyper.name = org.apache.spark.repl.SparkIMain$exprTyper\nlogger.sparkReplTyper.level = INFO\n\nlogger.sparkReplInterpreter.name = org.apache.spark.repl.SparkILoop$SparkILoopInterpreter\nlogger.sparkReplInterpreter.level = INFO\n\nlogger.jaegertracing.name = io.jaegertracing.spark\nlogger.jaegertracing.level = INFO\n\n# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support\nlogger.hiveMetastore.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler\nlogger.hiveMetastore.level = FATAL\n\nlogger.hiveFunctionRegistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry\nlogger.hiveFunctionRegistry.level = ERROR\n" }, { "path": "jaeger-spark-dependencies-elasticsearch/src/test/java/io/jaegertracing/spark/dependencies/elastic/ElasticsearchDependenciesDockerJobTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.elastic;\n\nimport io.jaegertracing.spark.dependencies.LogToConsolePrinter;\nimport org.testcontainers.containers.GenericContainer;\nimport org.testcontainers.utility.DockerImageName;\n\nimport java.util.concurrent.TimeUnit;\n\nimport static org.awaitility.Awaitility.await;\n\npublic class ElasticsearchDependenciesDockerJobTest extends ElasticsearchDependenciesJobTest {\n private static String dependenciesJobTag() {\n String tag = System.getenv(\"SPARK_DEPENDENCIES_JOB_IMAGE_TAG\");\n if (tag == null || tag.isEmpty()) {\n throw new IllegalStateException(\n \"SPARK_DEPENDENCIES_JOB_IMAGE_TAG environment variable is required but not set. \" +\n \"This variable must be set to ensure tests use the locally built Docker image.\");\n }\n return tag.trim();\n }\n\n @Override\n protected void deriveDependencies() {\n // Create the dependenciesJob instance so that after() method can call\n // indexDate() on it\n dependenciesJob = ElasticsearchDependenciesJob.builder()\n .nodes(\"http://\" + jaegerElasticsearchEnvironment.getElasticsearchIPPort())\n .day(java.time.LocalDate.now())\n .build();\n\n try {\n jaegerElasticsearchEnvironment.refresh();\n // Wait a bit to ensure all spans are fully indexed and visible\n Thread.sleep(2000);\n } catch (java.io.IOException e) {\n throw new RuntimeException(\"Could not refresh Elasticsearch\", e);\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n throw new RuntimeException(\"Interrupted while waiting\", e);\n }\n\n // Use the same date as the test - format it as ISO-8601 date string for the\n // DATE env var\n String dateStr = java.time.LocalDate.now().toString();\n\n System.out\n .println(\"Running Docker spark-dependencies job with DATE=\" + dateStr + \", ES_NODES=http://elasticsearch:9200\");\n System.out.println(\"::group::🚧 🚧 🚧 ElasticsearchDependenciesDockerJob logs\");\n try (GenericContainer sparkDependenciesJob = new GenericContainer<>(\n DockerImageName.parse(\"ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:\" + dependenciesJobTag()))\n .withNetwork(jaegerElasticsearchEnvironment.network)\n .withLogConsumer(new LogToConsolePrinter(\"[spark-dependencies] \"))\n .withEnv(\"STORAGE\", \"elasticsearch\")\n .withEnv(\"ES_NODES\", \"http://elasticsearch:9200\")\n .withEnv(\"DATE\", dateStr)\n .dependsOn(jaegerElasticsearchEnvironment.elasticsearch, jaegerElasticsearchEnvironment.jaegerAll)) {\n sparkDependenciesJob.start();\n await(\"spark-dependencies-job execution\")\n .atMost(3, TimeUnit.MINUTES)\n .until(() -> !sparkDependenciesJob.isRunning());\n } finally {\n System.out.println(\"::endgroup::\");\n }\n\n try {\n jaegerElasticsearchEnvironment.refresh();\n } catch (java.io.IOException e) {\n throw new RuntimeException(\"Could not refresh Elasticsearch\", e);\n }\n }\n}\n" }, { "path": "jaeger-spark-dependencies-elasticsearch/src/test/java/io/jaegertracing/spark/dependencies/elastic/ElasticsearchDependenciesJobTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.elastic;\n\n\nimport io.opentelemetry.api.trace.Span;\nimport io.opentelemetry.api.trace.Tracer;\nimport io.jaegertracing.spark.dependencies.test.DependenciesTest;\nimport io.jaegertracing.spark.dependencies.test.TracersGenerator;\nimport java.io.IOException;\nimport java.time.LocalDate;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.concurrent.TimeUnit;\nimport org.junit.After;\nimport org.junit.AfterClass;\nimport org.junit.Before;\nimport org.junit.BeforeClass;\nimport org.testcontainers.containers.wait.strategy.HttpWaitStrategy;\n\n/**\n * @author Pavol Loffay\n */\npublic class ElasticsearchDependenciesJobTest extends DependenciesTest {\n\n protected ElasticsearchDependenciesJob dependenciesJob;\n static JaegerElasticsearchEnvironment jaegerElasticsearchEnvironment;\n\n @BeforeClass\n public static void beforeClass() {\n jaegerElasticsearchEnvironment = new JaegerElasticsearchEnvironment();\n jaegerElasticsearchEnvironment.start(new HashMap<>(), jaegerVersion(), JaegerElasticsearchEnvironment.elasticsearchVersion());\n collectorUrl = jaegerElasticsearchEnvironment.getCollectorUrl();\n queryUrl = jaegerElasticsearchEnvironment.getQueryUrl();\n }\n\n @Before\n public void before() {\n String serviceName = UUID.randomUUID().toString();\n String operationName = UUID.randomUUID().toString();\n TracersGenerator.Tuple tuple = TracersGenerator.createJaeger(serviceName, collectorUrl);\n Tracer initStorageTracer = tuple.getA();\n Span span = initStorageTracer.spanBuilder(operationName).startSpan();\n span.setAttribute(\"foo\", \"bar\");\n span.end();\n tuple.getB().flush();\n try {\n // Give extra time for spans to be exported and indexed\n TimeUnit.SECONDS.sleep(2);\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n }\n waitJaegerQueryContains(serviceName, \"foo\");\n }\n\n @After\n public void after() throws IOException {\n if (dependenciesJob != null) {\n jaegerElasticsearchEnvironment.cleanUp(dependenciesJob.indexDate(\"jaeger-span\"), dependenciesJob.indexDate(\"jaeger-dependencies\"));\n }\n }\n\n @AfterClass\n public static void afterClass() {\n jaegerElasticsearchEnvironment.stop();\n }\n\n @Override\n protected void deriveDependencies() {\n dependenciesJob = ElasticsearchDependenciesJob.builder()\n .nodes(\"http://\" + jaegerElasticsearchEnvironment.getElasticsearchIPPort())\n .day(LocalDate.now())\n .build();\n try {\n jaegerElasticsearchEnvironment.refresh();\n } catch (IOException e) {\n throw new RuntimeException(\"Could not refresh Elasticsearch\", e);\n }\n dependenciesJob.run(\"peer.service\");\n try {\n jaegerElasticsearchEnvironment.refresh();\n } catch (IOException e) {\n throw new RuntimeException(\"Could not refresh Elasticsearch\", e);\n }\n }\n\n @Override\n protected void waitBetweenTraces() throws InterruptedException {\n try {\n jaegerElasticsearchEnvironment.refresh();\n } catch (IOException e) {\n throw new RuntimeException(\"Could not refresh Elasticsearch\", e);\n }\n }\n\n public static class BoundPortHttpWaitStrategy extends HttpWaitStrategy {\n private final int port;\n\n public BoundPortHttpWaitStrategy(int port) {\n this.port = port;\n }\n\n @Override\n protected Set getLivenessCheckPorts() {\n int mapptedPort = this.waitStrategyTarget.getMappedPort(port);\n return Collections.singleton(mapptedPort);\n }\n }\n}\n" }, { "path": "jaeger-spark-dependencies-elasticsearch/src/test/java/io/jaegertracing/spark/dependencies/elastic/ElasticsearchDependenciesTagFieldsJobTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.elastic;\n\nimport java.util.HashMap;\nimport org.junit.BeforeClass;\n\n/**\n * @author Pavol Loffay\n */\npublic class ElasticsearchDependenciesTagFieldsJobTest extends ElasticsearchDependenciesJobTest {\n\n @BeforeClass\n public static void beforeClass() {\n jaegerElasticsearchEnvironment = new JaegerElasticsearchEnvironment();\n HashMap jaegerEnvSetting = new HashMap<>();\n jaegerEnvSetting.put(\"ES_TAGS__AS_FIELDS_ALL\", \"true\");\n jaegerElasticsearchEnvironment.start(jaegerEnvSetting, jaegerVersion(), JaegerElasticsearchEnvironment.elasticsearchVersion());\n collectorUrl = jaegerElasticsearchEnvironment.getCollectorUrl();\n queryUrl = jaegerElasticsearchEnvironment.getQueryUrl();\n }\n}\n" }, { "path": "jaeger-spark-dependencies-elasticsearch/src/test/java/io/jaegertracing/spark/dependencies/elastic/JaegerElasticsearchEnvironment.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.elastic;\n\nimport static io.jaegertracing.spark.dependencies.test.DependenciesTest.jaegerVersion;\n\nimport io.jaegertracing.spark.dependencies.elastic.ElasticsearchDependenciesJobTest.BoundPortHttpWaitStrategy;\nimport java.io.IOException;\nimport java.util.Map;\nimport java.util.Optional;\nimport okhttp3.MediaType;\nimport okhttp3.OkHttpClient;\nimport okhttp3.Request;\nimport okhttp3.RequestBody;\nimport okhttp3.Response;\nimport org.testcontainers.containers.GenericContainer;\nimport org.testcontainers.containers.Network;\n\n/**\n * @author Pavol Loffay\n */\npublic class JaegerElasticsearchEnvironment {\n\n private OkHttpClient okHttpClient = new OkHttpClient();\n Network network;\n GenericContainer elasticsearch;\n GenericContainer jaegerAll;\n\n /**\n * Set these in subclasses\n */\n private String queryUrl;\n private String collectorUrl;\n\n public static String elasticsearchVersion() {\n String version = System.getProperty(\"elasticsearch.version\", System.getenv(\"ELASTICSEARCH_VERSION\"));\n return version != null ? version : \"7.17.10\";\n }\n\n public void start(Map jaegerEnvs, String jaegerVersion, String elasticsearchVersion) {\n network = Network.newNetwork();\n elasticsearch = new GenericContainer<>(String.format(\"docker.elastic.co/elasticsearch/elasticsearch:%s\", elasticsearchVersion))\n .withNetwork(network)\n .withNetworkAliases(\"elasticsearch\")\n .waitingFor(new BoundPortHttpWaitStrategy(9200).forStatusCode(200))\n .withExposedPorts(9200, 9300)\n .withEnv(\"xpack.security.enabled\", \"false\")\n .withEnv(\"discovery.type\", \"single-node\")\n .withEnv(\"network.bind_host\", \"elasticsearch\")\n .withEnv(\"network.host\", \"_site_\")\n .withEnv(\"network.publish_host\", \"_local_\");\n elasticsearch.start();\n\n jaegerAll = new GenericContainer<>(\"jaegertracing/jaeger:\" + jaegerVersion)\n .withNetwork(network)\n .withClasspathResourceMapping(\"jaeger-v2-config-elasticsearch.yaml\", \"/etc/jaeger/config.yaml\", org.testcontainers.containers.BindMode.READ_ONLY)\n .withCommand(\"--config\", \"/etc/jaeger/config.yaml\")\n .withEnv(jaegerEnvs)\n .waitingFor(new BoundPortHttpWaitStrategy(16687).forStatusCodeMatching(statusCode -> statusCode >= 200 && statusCode < 300))\n .withExposedPorts(16687, 16686, 4317, 4318, 14268, 9411);\n jaegerAll.start();\n\n collectorUrl = String.format(\"http://%s:%d\", jaegerAll.getContainerIpAddress(), jaegerAll.getMappedPort(4317));\n queryUrl = String.format(\"http://%s:%d\", jaegerAll.getContainerIpAddress(), jaegerAll.getMappedPort(16686));\n }\n\n public void cleanUp(String[] spanIndex, String[] dependenciesIndex) throws IOException {\n String matchAllQuery = \"{\\\"query\\\": {\\\"match_all\\\":{} }}\";\n Request request = new Request.Builder()\n .url(String.format(\"http://%s:%d/%s,%s/_delete_by_query?conflicts=proceed\",\n elasticsearch.getContainerIpAddress(),\n elasticsearch.getMappedPort(9200),\n // we don't use index prefix\n spanIndex[0],\n dependenciesIndex[0]))\n .post(\n RequestBody.create(MediaType.parse(\"application/json; charset=utf-8\"), matchAllQuery))\n .build();\n\n\n try (Response response = okHttpClient.newCall(request).execute()) {\n if (!response.isSuccessful()) {\n String body = response.body().string();\n throw new IllegalStateException(String.format(\"Could not remove data from ES: %s, %s\", response, body));\n }\n }\n }\n\n /**\n * In Elasticsearch, the _refresh endpoint is used to make recently indexed,\n * updated, or deleted documents visible to search, as otherwise they might\n * be still sitting in a memory buffer.\n */\n public void refresh() throws IOException {\n Request request = new Request.Builder()\n .url(String.format(\"http://%s:%d/_refresh\",\n elasticsearch.getContainerIpAddress(),\n elasticsearch.getMappedPort(9200)))\n .post(RequestBody.create(MediaType.parse(\"application/json; charset=utf-8\"), \"\"))\n .build();\n\n try (Response response = okHttpClient.newCall(request).execute()) {\n if (!response.isSuccessful()) {\n String body = response.body().string();\n throw new IllegalStateException(String.format(\"Could not refresh ES: %s, %s\", response, body));\n }\n }\n }\n\n public void stop() {\n Optional.of(jaegerAll).ifPresent(GenericContainer::close);\n Optional.of(elasticsearch).ifPresent(GenericContainer::close);\n Optional.of(network).ifPresent(network1 -> {\n try {\n network1.close();\n } catch (Exception e) {\n e.printStackTrace();\n }\n });\n }\n\n public String getQueryUrl() {\n return queryUrl;\n }\n\n public String getCollectorUrl() {\n return collectorUrl;\n }\n\n public String getElasticsearchIPPort() {\n return String.format(\"%s:%d\", elasticsearch.getContainerIpAddress(), elasticsearch.getMappedPort(9200));\n }\n}\n" }, { "path": "jaeger-spark-dependencies-elasticsearch/src/test/resources/jaeger-v2-config-elasticsearch.yaml", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\nservice:\n extensions: [jaeger_storage, jaeger_query, healthcheckv2]\n pipelines:\n traces:\n receivers: [otlp, jaeger, zipkin]\n processors: [filter/jaeger, batch]\n exporters: [jaeger_storage_exporter]\n telemetry:\n resource:\n service.name: jaeger-backend\n metrics:\n level: detailed\n readers:\n - pull:\n exporter:\n prometheus:\n host: 0.0.0.0\n port: 8888\n logs:\n level: info\n traces:\n level: none\n\nextensions:\n healthcheckv2:\n use_v2: true\n http:\n endpoint: \"0.0.0.0:16687\"\n status:\n enabled: true\n path: \"/\"\n\n jaeger_query:\n storage:\n traces: some_storage\n\n jaeger_storage:\n backends:\n some_storage:\n elasticsearch:\n server_urls:\n - http://elasticsearch:9200\n service_cache_ttl: 1s\n\nreceivers:\n otlp:\n protocols:\n grpc:\n endpoint: \"0.0.0.0:4317\"\n http:\n endpoint: \"0.0.0.0:4318\"\n\n jaeger:\n protocols:\n grpc:\n thrift_binary:\n thrift_compact:\n thrift_http:\n endpoint: \"0.0.0.0:14268\"\n\n zipkin:\n endpoint: \"0.0.0.0:9411\"\n\nprocessors:\n filter/jaeger:\n error_mode: ignore\n traces:\n span:\n - 'resource.attributes[\"service.name\"] == \"jaeger\"'\n \n batch:\n\nexporters:\n jaeger_storage_exporter:\n trace_storage: some_storage\n" }, { "path": "jaeger-spark-dependencies-opensearch/pom.xml", "content": "\n\n\n 4.0.0\n \n jaeger-spark-dependencies-parent\n io.jaegertracing.dependencies\n 0.0.1-SNAPSHOT\n \n\n jaeger-spark-dependencies-opensearch\n\n \n \n ${project.groupId}\n jaeger-spark-dependencies-common\n \n \n com.fasterxml.jackson.core\n jackson-annotations\n \n \n \n\n \n org.opensearch.client\n opensearch-spark-30_${version.scala.binary}\n 1.3.0\n \n \n org.apache.spark\n spark-core_2.12\n \n \n org.apache.spark\n spark-sql_2.12\n \n \n \n\n \n org.opensearch.client\n opensearch-rest-high-level-client\n 2.18.0\n \n\n \n ${project.groupId}\n jaeger-spark-dependencies-test\n test\n \n \n org.testcontainers\n testcontainers\n test\n \n \n org.opensearch\n opensearch-testcontainers\n 2.1.4\n test\n \n \n org.awaitility\n awaitility\n ${version.org.awaitility-awaitility}\n test\n \n \n io.opentelemetry\n opentelemetry-api\n ${version.io.opentelemetry}\n test\n \n \n io.jaegertracing\n jaeger-client\n ${version.io.jaegertracing}\n test\n \n \n\n \n \n \n maven-shade-plugin\n ${version.maven-shade-plugin}\n \n \n package\n \n shade\n \n \n \n \n reference.conf\n \n \n io.jaegertracing.spark.dependencies.opensearch.OpenSearchDependenciesJob\n \n \n false\n \n \n *:*\n \n META-INF/*.SF\n META-INF/*.DSA\n META-INF/*.RSA\n \n \n \n false\n \n \n \n \n \n \n\n" }, { "path": "jaeger-spark-dependencies-opensearch/src/main/java/io/jaegertracing/spark/dependencies/opensearch/OpenSearchDependenciesJob.java", "content": "/**\n * Copyright 2017 The Jaeger Authors\n * Copyright 2016-2017 The OpenZipkin Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\npackage io.jaegertracing.spark.dependencies.opensearch;\n\nimport com.fasterxml.jackson.core.JsonProcessingException;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport io.jaegertracing.spark.dependencies.DependenciesSparkHelper;\nimport io.jaegertracing.spark.dependencies.Utils;\nimport io.jaegertracing.spark.dependencies.model.Dependency;\nimport io.jaegertracing.spark.dependencies.model.Span;\nimport java.time.LocalDate;\nimport java.time.ZoneOffset;\nimport java.time.ZonedDateTime;\nimport java.time.format.DateTimeFormatter;\nimport java.util.Collections;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.regex.Pattern;\nimport org.apache.spark.SparkConf;\nimport org.apache.spark.api.java.JavaPairRDD;\nimport org.apache.spark.api.java.JavaSparkContext;\nimport org.opensearch.spark.rdd.api.java.JavaOpenSearchSpark;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n * @author OpenZipkin authors\n * @author Pavol Loffay\n * @author Danish Siddiqui\n */\npublic class OpenSearchDependenciesJob {\n private static final Logger log = LoggerFactory.getLogger(OpenSearchDependenciesJob.class);\n private static final Pattern PORT_PATTERN = Pattern.compile(\":\\\\d+\");\n\n public static Builder builder() {\n return new Builder();\n }\n\n public static final class Builder {\n\n String hosts = Utils.getEnv(\"OS_NODES\", \"127.0.0.1\");\n String username = Utils.getEnv(\"OS_USERNAME\", null);\n String password = Utils.getEnv(\"OS_PASSWORD\", null);\n Boolean clientNodeOnly = Boolean.parseBoolean(Utils.getEnv(\"OS_CLIENT_NODE_ONLY\", \"false\"));\n Boolean nodesWanOnly = Boolean.parseBoolean(Utils.getEnv(\"OS_NODES_WAN_ONLY\", \"false\"));\n String indexPrefix = Utils.getEnv(\"OS_INDEX_PREFIX\", null);\n String indexDatePattern = datePattern(Utils.getEnv(\"OS_INDEX_DATE_SEPARATOR\", \"-\"));\n String spanRange = Utils.getEnv(\"OS_TIME_RANGE\", \"24h\");\n Boolean useAliases = Boolean.parseBoolean(Utils.getEnv(\"OS_USE_ALIASES\", \"false\"));\n Boolean allowSelfSigned = Boolean.parseBoolean(Utils.getEnv(\"OS_SSL_CERT_ALLOW_SELF_SIGNED\", \"false\"));\n\n final Map sparkProperties = new LinkedHashMap<>();\n\n Builder() {\n sparkProperties.put(\"spark.ui.enabled\", \"false\");\n // don't die if there are no spans\n sparkProperties.put(\"opensearch.index.read.missing.as.empty\", \"true\");\n sparkProperties.put(\"opensearch.net.ssl.keystore.location\",\n getSystemPropertyAsFileResource(\"javax.net.ssl.keyStore\"));\n sparkProperties.put(\"opensearch.net.ssl.keystore.pass\",\n System.getProperty(\"javax.net.ssl.keyStorePassword\", \"\"));\n sparkProperties.put(\"opensearch.net.ssl.truststore.location\",\n getSystemPropertyAsFileResource(\"javax.net.ssl.trustStore\"));\n sparkProperties.put(\"opensearch.net.ssl.truststore.pass\",\n System.getProperty(\"javax.net.ssl.trustStorePassword\", \"\"));\n if (allowSelfSigned) {\n sparkProperties.put(\"opensearch.net.ssl.cert.allow.self.signed\", \"true\");\n }\n\n }\n\n // local[*] master lets us run & test the job locally without setting a Spark cluster\n String sparkMaster = Utils.getEnv(\"SPARK_MASTER\", \"local[*]\");\n // needed when not in local mode\n String[] jars;\n\n // By default the job only works on traces whose first timestamp is today\n ZonedDateTime day = ZonedDateTime.of(LocalDate.now().atStartOfDay(), ZoneOffset.UTC);\n\n /** When set, this indicates which jars to distribute to the cluster. */\n public Builder jars(String... jars) {\n this.jars = jars;\n return this;\n }\n\n /** opensearch.nodes separated by ',' */\n public Builder nodes(String hosts) {\n Utils.checkNoTNull(hosts, \"nodes\");\n this.hosts = hosts;\n this.nodesWanOnly = true;\n return this;\n }\n\n /** username used for basic auth. Needed when Shield or X-Pack security is enabled */\n public Builder username(String username) {\n this.username = username;\n return this;\n }\n\n /** password used for basic auth. Needed when Shield or X-Pack security is enabled */\n public Builder password(String password) {\n this.password = password;\n return this;\n }\n\n /** index prefix for Jaeger indices. By default empty */\n public Builder indexPrefix(String indexPrefix) {\n this.indexPrefix = indexPrefix;\n return this;\n }\n\n /** index date pattern for Jaeger indices. By default yyyy-MM-dd */\n public Builder indexDatePattern(String indexDatePattern) {\n this.indexDatePattern = indexDatePattern;\n return this;\n }\n\n /** span range for Jaeger indices. By default 24h */\n public Builder spanRange(String spanRange) {\n this.spanRange = spanRange;\n return this;\n }\n\n /** Day to process dependencies for. Defaults to today. */\n public Builder day(LocalDate day) {\n this.day = day.atStartOfDay(ZoneOffset.UTC);\n return this;\n }\n\n /** Whether the connector is used against an OpenSearch instance in a cloud/restricted\n * environment over the WAN, such as Amazon Web Services. In this mode, the\n * connector disables discovery and only connects through the declared opensearch.nodes during all operations,\n * including reads and writes. Note that in this mode, performance is highly affected. */\n public Builder nodesWanOnly(boolean wanOnly) {\n this.nodesWanOnly = wanOnly;\n return this;\n }\n\n private static void logIfNoPort(String hosts) {\n if (!PORT_PATTERN.matcher(hosts).find()) {\n log.warn(\"Port is not specified, default port 9200 will be used\");\n }\n }\n\n public OpenSearchDependenciesJob build() {\n String hosts = System.getenv(\"OS_NODES\");\n String wanOnly = System.getenv(\"OS_NODES_WAN_ONLY\");\n // Optimize user configuration - nodes specified but wan only not\n if (hosts != null && wanOnly == null) {\n this.nodesWanOnly = true;\n }\n logIfNoPort(this.hosts);\n return new OpenSearchDependenciesJob(this);\n }\n }\n\n private static String getSystemPropertyAsFileResource(String key) {\n String prop = System.getProperty(key, \"\");\n return prop != null && !prop.isEmpty() ? \"file:\" + prop : prop;\n }\n\n private final ZonedDateTime day;\n private final SparkConf conf;\n private final String indexPrefix;\n private final String indexDatePattern;\n private final String spanRange;\n private final Boolean useAliases;\n\n OpenSearchDependenciesJob(Builder builder) {\n this.day = builder.day;\n this.conf = new SparkConf(true).setMaster(builder.sparkMaster).setAppName(getClass().getName());\n if (builder.jars != null) {\n conf.setJars(builder.jars);\n }\n if (builder.username != null) {\n conf.set(\"opensearch.net.http.auth.user\", builder.username);\n }\n if (builder.password != null) {\n conf.set(\"opensearch.net.http.auth.pass\", builder.password);\n }\n conf.set(\"opensearch.nodes\", builder.hosts);\n if (builder.hosts.indexOf(\"https\") != -1) {\n conf.set(\"opensearch.net.ssl\", \"true\");\n }\n if (builder.nodesWanOnly) {\n conf.set(\"opensearch.nodes.wan.only\", \"true\");\n }\n if (builder.clientNodeOnly) {\n conf.set(\"opensearch.nodes.discovery\", \"0\");\n conf.set(\"opensearch.nodes.client.only\", \"1\");\n }\n for (Map.Entry entry : builder.sparkProperties.entrySet()) {\n conf.set(entry.getKey(), entry.getValue());\n }\n this.indexPrefix = builder.indexPrefix;\n this.indexDatePattern = builder.indexDatePattern;\n this.spanRange = builder.spanRange;\n this.useAliases = builder.useAliases;\n }\n\n /**\n * https://github.com/jaegertracing/jaeger/blob/master/CHANGELOG.md#190-2019-01-21\n */\n private static String prefixBefore19(String prefix) {\n return prefix != null ? String.format(\"%s:\", prefix) : \"\";\n }\n\n private static String prefix(String prefix) {\n return prefix != null ? String.format(\"%s-\", prefix) : \"\";\n }\n\n private static String datePattern(String separator) {\n if (separator.equals(\"\")) {\n return \"yyyyMMdd\";\n }\n // ' is escape character in date format, we should double it here.\n if (separator.contains(\"'\")) {\n separator = separator.replace(\"'\", \"''\");\n }\n return String.format(\"yyyy'%s'MM'%s'dd\", separator, separator);\n }\n\n public void run(String peerServiceTag) {\n\n String[] readIndices;\n String[] writeIndex;\n\n // use alias indices common when using index rollover\n if (this.useAliases) {\n readIndices = new String[]{prefix(indexPrefix) + \"jaeger-span-read\", prefixBefore19(indexPrefix) + \"jaeger-span-read\"};\n writeIndex = new String[] {prefix(indexPrefix) + \"jaeger-dependencies-write\", prefixBefore19(indexPrefix) + \"jaeger-dependencies-write\"};\n }\n else {\n readIndices = indexDate(\"jaeger-span\");\n writeIndex = indexDate(\"jaeger-dependencies\");\n }\n\n run(readIndices, writeIndex, peerServiceTag);\n }\n\n String[] indexDate(String index) {\n String date = day.toLocalDate().format(DateTimeFormatter.ofPattern(indexDatePattern));\n if (indexPrefix != null && indexPrefix.length() > 0) {\n return new String[]{String.format(\"%s%s-%s\", prefix(indexPrefix), index, date), String.format(\"%s%s-%s\", prefixBefore19(indexPrefix), index, date)};\n }\n // if there is no prefix we read and write only to one index\n return new String[]{String.format(\"%s-%s\", index, date)};\n }\n\n void run(String[] spanIndices, String[] depIndices,String peerServiceTag) {\n JavaSparkContext sc = new JavaSparkContext(conf);\n try {\n for (int i = 0; i < spanIndices.length; i++) {\n String spanIndex = spanIndices[i];\n String depIndex = depIndices[i];\n log.info(\"Running Dependencies job for {}, reading from {} index, result storing to {}\", day, spanIndex, depIndex);\n // Send raw query to OS to select only the docs / spans we want to consider for this job\n // This doesn't change the default behavior as the daily indexes only contain up to 24h of data\n String osQuery = String.format(\"{\\\"range\\\": {\\\"startTimeMillis\\\": { \\\"gte\\\": \\\"now-%s\\\" }}}\", spanRange);\n JavaPairRDD> traces = JavaOpenSearchSpark.opensearchRDD(sc, spanIndex, osQuery)\n .map(new OpenSearchTupleToSpan())\n .groupBy(Span::getTraceId);\n List dependencyLinks = DependenciesSparkHelper.derive(traces,peerServiceTag);\n \n // No version check needed for OpenSearch as we don't support types in indexes\n store(sc, dependencyLinks, depIndex);\n log.info(\"Done, {} dependency objects created\", dependencyLinks.size());\n if (dependencyLinks.size() > 0) {\n // we do not derive dependencies for old prefix \"prefix:\" if new prefix \"prefix-\" contains data\n break;\n }\n }\n } finally {\n sc.stop();\n }\n }\n\n private void store(JavaSparkContext javaSparkContext, List dependencyLinks, String resource) {\n if (dependencyLinks.isEmpty()) {\n return;\n }\n\n String json;\n try {\n ObjectMapper objectMapper = new ObjectMapper();\n json = objectMapper.writeValueAsString(new OpenSearchDependencies(dependencyLinks, day));\n } catch (JsonProcessingException e) {\n throw new IllegalStateException(\"Could not serialize dependencies\", e);\n }\n\n JavaOpenSearchSpark.saveJsonToOpenSearch(javaSparkContext.parallelize(Collections.singletonList(json)), resource);\n }\n\n /**\n * Helper class used to serialize dependencies to JSON.\n */\n public static final class OpenSearchDependencies {\n private List dependencies;\n private ZonedDateTime ts;\n\n public OpenSearchDependencies(List dependencies, ZonedDateTime ts) {\n this.dependencies = dependencies;\n this.ts = ts;\n }\n\n public List getDependencies() {\n return dependencies;\n }\n\n public String getTimestamp() {\n // Jaeger OS dependency storage uses RFC3339Nano for timestamp\n return ts.format(DateTimeFormatter.ofPattern(\"yyyy-MM-dd'T'HH:mm:ssXXX\"));\n }\n }\n\n /**\n * Entry point for running OpenSearchDependenciesJob directly.\n */\n public static void main(String[] args) throws java.io.UnsupportedEncodingException {\n LocalDate date = LocalDate.now();\n if (args.length == 1) {\n date = LocalDate.parse(args[0]);\n } else if (System.getenv(\"DATE\") != null) {\n date = LocalDate.parse(System.getenv(\"DATE\"));\n }\n\n String peerServiceTag = System.getenv(\"PEER_SERVICE_TAG\");\n if (peerServiceTag == null) {\n peerServiceTag = \"peer.service\";\n }\n\n String jarPath = Utils.pathToUberJar(OpenSearchDependenciesJob.class);\n OpenSearchDependenciesJob.builder()\n .jars(jarPath)\n .day(date)\n .build()\n .run(peerServiceTag);\n }\n}\n" }, { "path": "jaeger-spark-dependencies-opensearch/src/main/java/io/jaegertracing/spark/dependencies/opensearch/OpenSearchTupleToSpan.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.opensearch;\n\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport io.jaegertracing.spark.dependencies.json.JsonHelper;\nimport io.jaegertracing.spark.dependencies.model.Span;\nimport java.util.Map;\nimport org.apache.spark.api.java.function.Function;\nimport scala.Tuple2;\n\n/**\n * @author Pavol Loffay\n * @author Danish Siddiqui\n */\npublic class OpenSearchTupleToSpan implements Function>, Span> {\n\n private ObjectMapper objectMapper = JsonHelper.configure(new ObjectMapper());\n\n @Override\n public Span call(Tuple2> tuple) throws Exception {\n Span span = objectMapper.convertValue(tuple._2(), Span.class);\n String originalTraceId = span.getTraceId();\n span.setTraceId(normalizeTraceId(originalTraceId));\n if (span.getTags() != null) {\n span.getTags().sort((o1, o2) -> o1.getKey().compareTo(o2.getKey()));\n }\n if (span.getRefs() != null) {\n span.getRefs().sort((o1, o2) -> o1.getSpanId().compareTo(o2.getSpanId()));\n }\n\n return span;\n }\n\n private String normalizeTraceId(String traceId) {\n if (traceId != null && traceId.length() < 32) {\n return String.format(\"%32s\", traceId).replace(' ', '0');\n }\n return traceId;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-opensearch/src/main/resources/log4j.properties", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n# Set everything to be logged to the console\nlog4j.rootCategory=WARN, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlog4j.logger.org.spark-project.jetty=WARN\nlog4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO\nlog4j.logger.io.jaegertracing.spark=INFO\n\n# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support\nlog4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL\nlog4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR\n" }, { "path": "jaeger-spark-dependencies-opensearch/src/main/resources/log4j2.component.properties", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n# Disable Log4j status logger console output\nlog4j2.StatusLogger.level = OFF\n" }, { "path": "jaeger-spark-dependencies-opensearch/src/main/resources/log4j2.properties", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\n# Set root logger level to WARN and use console appender\nrootLogger.level = WARN\nrootLogger.appenderRef.console.ref = console\n\n# Console appender configuration\nappender.console.type = Console\nappender.console.name = console\nappender.console.target = SYSTEM_ERR\nappender.console.layout.type = PatternLayout\nappender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlogger.jetty.name = org.spark-project.jetty\nlogger.jetty.level = WARN\n\nlogger.jettyLifecycle.name = org.spark-project.jetty.util.component.AbstractLifeCycle\nlogger.jettyLifecycle.level = ERROR\n\nlogger.sparkReplTyper.name = org.apache.spark.repl.SparkIMain$exprTyper\nlogger.sparkReplTyper.level = INFO\n\nlogger.sparkReplInterpreter.name = org.apache.spark.repl.SparkILoop$SparkILoopInterpreter\nlogger.sparkReplInterpreter.level = INFO\n\nlogger.jaegertracing.name = io.jaegertracing.spark\nlogger.jaegertracing.level = INFO\n\n# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support\nlogger.hiveMetastore.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler\nlogger.hiveMetastore.level = FATAL\n\nlogger.hiveFunctionRegistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry\nlogger.hiveFunctionRegistry.level = ERROR\n" }, { "path": "jaeger-spark-dependencies-opensearch/src/test/java/io/jaegertracing/spark/dependencies/opensearch/JaegerOpenSearchEnvironment.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.opensearch;\n\nimport java.io.IOException;\nimport java.util.Map;\nimport java.util.Optional;\nimport okhttp3.MediaType;\nimport okhttp3.OkHttpClient;\nimport okhttp3.Request;\nimport okhttp3.RequestBody;\nimport okhttp3.Response;\nimport org.testcontainers.containers.GenericContainer;\nimport org.testcontainers.containers.Network;\n\n/**\n * @author Pavol Loffay\n * @author Danish Siddiqui\n */\npublic class JaegerOpenSearchEnvironment {\n\n private OkHttpClient okHttpClient = new OkHttpClient();\n Network network;\n GenericContainer opensearch;\n GenericContainer jaegerAll;\n\n /**\n * Set these in subclasses\n */\n private String queryUrl;\n private String collectorUrl;\n\n public static String opensearchVersion() {\n String version = System.getProperty(\"opensearch.version\", System.getenv(\"OPENSEARCH_VERSION\"));\n return version != null ? version : \"2.11.1\";\n }\n\n public void start(Map jaegerEnvs, String jaegerVersion, String opensearchVersion) {\n network = Network.newNetwork();\n opensearch = new GenericContainer<>(String.format(\"opensearchproject/opensearch:%s\", opensearchVersion))\n .withNetwork(network)\n .withNetworkAliases(\"opensearch\")\n .waitingFor(new BoundPortHttpWaitStrategy(9200).forStatusCode(200))\n .withExposedPorts(9200)\n .withEnv(\"DISABLE_SECURITY_PLUGIN\", \"true\")\n .withEnv(\"discovery.type\", \"single-node\")\n .withEnv(\"network.bind_host\", \"opensearch\")\n .withEnv(\"network.host\", \"0.0.0.0\");\n opensearch.start();\n\n jaegerAll = new GenericContainer<>(\"jaegertracing/jaeger:\" + jaegerVersion)\n .withNetwork(network)\n .withClasspathResourceMapping(\"jaeger-v2-config-opensearch.yaml\", \"/etc/jaeger/config.yaml\",\n org.testcontainers.containers.BindMode.READ_ONLY)\n .withCommand(\"--config\", \"/etc/jaeger/config.yaml\")\n .withEnv(jaegerEnvs)\n .waitingFor(new BoundPortHttpWaitStrategy(16687)\n .forStatusCodeMatching(statusCode -> statusCode >= 200 && statusCode < 300))\n .withExposedPorts(16687, 16686, 4317, 4318, 14268, 9411);\n jaegerAll.start();\n\n collectorUrl = String.format(\"http://%s:%d\", jaegerAll.getHost(), jaegerAll.getMappedPort(4317));\n queryUrl = String.format(\"http://%s:%d\", jaegerAll.getHost(), jaegerAll.getMappedPort(16686));\n }\n\n public void cleanUp(String[] spanIndex, String[] dependenciesIndex) throws IOException {\n String matchAllQuery = \"{\\\"query\\\": {\\\"match_all\\\":{} }}\";\n Request request = new Request.Builder()\n .url(String.format(\"http://%s:%d/%s,%s/_delete_by_query?conflicts=proceed\",\n opensearch.getHost(),\n opensearch.getMappedPort(9200),\n // we don't use index prefix\n spanIndex[0],\n dependenciesIndex[0]))\n .post(\n RequestBody.create(MediaType.parse(\"application/json; charset=utf-8\"), matchAllQuery))\n .build();\n\n try (Response response = okHttpClient.newCall(request).execute()) {\n if (!response.isSuccessful()) {\n String body = response.body().string();\n throw new IllegalStateException(String.format(\"Could not remove data from OS: %s, %s\", response, body));\n }\n }\n }\n\n /**\n * In OpenSearch, the _refresh endpoint is used to make recently indexed,\n * updated, or deleted documents visible to search, as otherwise they might\n * be still sitting in a memory buffer.\n */\n public void refresh() throws IOException {\n Request request = new Request.Builder()\n .url(String.format(\"http://%s:%d/_refresh\",\n opensearch.getHost(),\n opensearch.getMappedPort(9200)))\n .post(RequestBody.create(MediaType.parse(\"application/json; charset=utf-8\"), \"\"))\n .build();\n\n try (Response response = okHttpClient.newCall(request).execute()) {\n if (!response.isSuccessful()) {\n String body = response.body().string();\n throw new IllegalStateException(String.format(\"Could not refresh OS: %s, %s\", response, body));\n }\n }\n }\n\n public void stop() {\n Optional.of(jaegerAll).ifPresent(GenericContainer::close);\n Optional.of(opensearch).ifPresent(GenericContainer::close);\n Optional.of(network).ifPresent(network1 -> {\n try {\n network1.close();\n } catch (Exception e) {\n e.printStackTrace();\n }\n });\n }\n\n public String getQueryUrl() {\n return queryUrl;\n }\n\n public String getCollectorUrl() {\n return collectorUrl;\n }\n\n public String getOpenSearchIPPort() {\n return String.format(\"%s:%d\", opensearch.getHost(), opensearch.getMappedPort(9200));\n }\n\n public static class BoundPortHttpWaitStrategy extends org.testcontainers.containers.wait.strategy.HttpWaitStrategy {\n private final int port;\n\n public BoundPortHttpWaitStrategy(int port) {\n this.port = port;\n }\n\n @Override\n protected java.util.Set getLivenessCheckPorts() {\n int mapptedPort = this.waitStrategyTarget.getMappedPort(port);\n return java.util.Collections.singleton(mapptedPort);\n }\n }\n}\n" }, { "path": "jaeger-spark-dependencies-opensearch/src/test/java/io/jaegertracing/spark/dependencies/opensearch/OpenSearchDependenciesDockerJobTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.opensearch;\n\nimport io.jaegertracing.spark.dependencies.LogToConsolePrinter;\nimport org.testcontainers.containers.GenericContainer;\n\n/**\n * @author Danish Siddiqui\n */\nimport org.testcontainers.utility.DockerImageName;\n\nimport java.util.concurrent.TimeUnit;\n\nimport static org.awaitility.Awaitility.await;\n\npublic class OpenSearchDependenciesDockerJobTest extends OpenSearchDependenciesJobTest {\n private static String dependenciesJobTag() {\n String tag = System.getenv(\"SPARK_DEPENDENCIES_JOB_IMAGE_TAG\");\n if (tag == null || tag.trim().isEmpty()) {\n throw new IllegalStateException(\n \"SPARK_DEPENDENCIES_JOB_IMAGE_TAG environment variable is required but not set. \" +\n \"This variable must be set to ensure tests use the locally built Docker image.\");\n }\n return tag.trim();\n }\n\n @Override\n protected void deriveDependencies() {\n // Create the dependenciesJob instance so that after() method can call\n // indexDate() on it\n dependenciesJob = OpenSearchDependenciesJob.builder()\n .nodes(\"http://\" + jaegerOpenSearchEnvironment.getOpenSearchIPPort())\n .day(java.time.LocalDate.now())\n .build();\n\n try {\n jaegerOpenSearchEnvironment.refresh();\n // Wait a bit to ensure all spans are fully indexed and visible\n Thread.sleep(2000);\n } catch (java.io.IOException e) {\n throw new RuntimeException(\"Could not refresh OpenSearch\", e);\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n throw new RuntimeException(\"Interrupted while waiting\", e);\n }\n\n // Use the same date as the test - format it as ISO-8601 date string for the\n // DATE env var\n String dateStr = java.time.LocalDate.now().toString();\n\n System.out\n .println(\"Running Docker spark-dependencies job with DATE=\" + dateStr + \", OS_NODES=http://opensearch:9200\");\n System.out.println(\"::group::🚧 🚧 🚧 OpenSearchDependenciesDockerJob logs\");\n try (GenericContainer sparkDependenciesJob = new GenericContainer<>(\n DockerImageName.parse(\"ghcr.io/jaegertracing/spark-dependencies/spark-dependencies:\" + dependenciesJobTag()))\n .withNetwork(jaegerOpenSearchEnvironment.network)\n .withLogConsumer(new LogToConsolePrinter(\"[spark-dependencies] \"))\n .withEnv(\"STORAGE\", \"opensearch\")\n .withEnv(\"OS_NODES\", \"http://opensearch:9200\")\n .withEnv(\"DATE\", dateStr)\n .dependsOn(jaegerOpenSearchEnvironment.opensearch, jaegerOpenSearchEnvironment.jaegerAll)) {\n sparkDependenciesJob.start();\n await(\"spark-dependencies-job execution\")\n .atMost(3, TimeUnit.MINUTES)\n .until(() -> !sparkDependenciesJob.isRunning());\n\n Long exitCode = sparkDependenciesJob.getCurrentContainerInfo()\n .getState()\n .getExitCodeLong();\n\n if (exitCode != null && exitCode != 0) {\n throw new RuntimeException(\"Spark dependencies job failed with exit code: \" + exitCode);\n }\n } finally {\n System.out.println(\"::endgroup::\");\n }\n\n try {\n jaegerOpenSearchEnvironment.refresh();\n } catch (java.io.IOException e) {\n throw new RuntimeException(\"Could not refresh OpenSearch\", e);\n }\n }\n}\n" }, { "path": "jaeger-spark-dependencies-opensearch/src/test/java/io/jaegertracing/spark/dependencies/opensearch/OpenSearchDependenciesJobTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.opensearch;\n\nimport io.opentelemetry.api.trace.Span;\nimport io.opentelemetry.api.trace.Tracer;\nimport io.jaegertracing.spark.dependencies.test.DependenciesTest;\nimport io.jaegertracing.spark.dependencies.test.TracersGenerator;\nimport java.io.IOException;\nimport java.time.LocalDate;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.Set;\nimport java.util.UUID;\nimport java.util.concurrent.TimeUnit;\nimport org.junit.After;\nimport org.junit.AfterClass;\nimport org.junit.Before;\nimport org.junit.BeforeClass;\nimport org.testcontainers.containers.wait.strategy.HttpWaitStrategy;\n\n/**\n * @author Pavol Loffay\n * @author Danish Siddiqui\n */\npublic class OpenSearchDependenciesJobTest extends DependenciesTest {\n\n protected OpenSearchDependenciesJob dependenciesJob;\n static JaegerOpenSearchEnvironment jaegerOpenSearchEnvironment;\n\n @BeforeClass\n public static void beforeClass() {\n jaegerOpenSearchEnvironment = new JaegerOpenSearchEnvironment();\n jaegerOpenSearchEnvironment.start(new HashMap<>(), jaegerVersion(),\n JaegerOpenSearchEnvironment.opensearchVersion());\n collectorUrl = jaegerOpenSearchEnvironment.getCollectorUrl();\n queryUrl = jaegerOpenSearchEnvironment.getQueryUrl();\n }\n\n @Before\n public void before() throws Exception {\n String serviceName = UUID.randomUUID().toString();\n String operationName = UUID.randomUUID().toString();\n TracersGenerator.Tuple tuple = TracersGenerator.createJaeger(serviceName,\n collectorUrl);\n Tracer initStorageTracer = tuple.getA();\n Span span = initStorageTracer.spanBuilder(operationName).startSpan();\n span.setAttribute(\"foo\", \"bar\");\n span.end();\n tuple.getB().flush();\n try {\n // Give extra time for spans to be exported and indexed\n TimeUnit.SECONDS.sleep(2);\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n }\n waitJaegerQueryContains(serviceName, \"foo\");\n }\n\n @After\n public void after() throws IOException {\n if (dependenciesJob != null) {\n jaegerOpenSearchEnvironment.cleanUp(dependenciesJob.indexDate(\"jaeger-span\"),\n dependenciesJob.indexDate(\"jaeger-dependencies\"));\n }\n }\n\n @AfterClass\n public static void afterClass() {\n jaegerOpenSearchEnvironment.stop();\n }\n\n @Override\n protected void deriveDependencies() {\n dependenciesJob = OpenSearchDependenciesJob.builder()\n .nodes(\"http://\" + jaegerOpenSearchEnvironment.getOpenSearchIPPort())\n .day(LocalDate.now())\n .build();\n try {\n jaegerOpenSearchEnvironment.refresh();\n } catch (IOException e) {\n throw new RuntimeException(\"Could not refresh OpenSearch\", e);\n }\n dependenciesJob.run(\"peer.service\");\n try {\n jaegerOpenSearchEnvironment.refresh();\n } catch (IOException e) {\n throw new RuntimeException(\"Could not refresh OpenSearch\", e);\n }\n }\n\n @Override\n protected void waitBetweenTraces() throws InterruptedException {\n try {\n jaegerOpenSearchEnvironment.refresh();\n } catch (IOException e) {\n throw new RuntimeException(\"Could not refresh OpenSearch\", e);\n }\n }\n\n public static class BoundPortHttpWaitStrategy extends HttpWaitStrategy {\n private final int port;\n\n public BoundPortHttpWaitStrategy(int port) {\n this.port = port;\n }\n\n @Override\n protected Set getLivenessCheckPorts() {\n int mapptedPort = this.waitStrategyTarget.getMappedPort(port);\n return Collections.singleton(mapptedPort);\n }\n }\n}\n" }, { "path": "jaeger-spark-dependencies-opensearch/src/test/resources/jaeger-v2-config-opensearch.yaml", "content": "#\n# Copyright (c) The Jaeger Authors\n# SPDX-License-Identifier: Apache-2.0\n#\n\nservice:\n extensions: [jaeger_storage, jaeger_query, healthcheckv2]\n pipelines:\n traces:\n receivers: [otlp, jaeger, zipkin]\n processors: [filter/jaeger, batch]\n exporters: [jaeger_storage_exporter]\n telemetry:\n resource:\n service.name: jaeger-backend\n metrics:\n \n level: detailed\n readers:\n - pull:\n exporter:\n prometheus:\n host: 0.0.0.0\n port: 8888\n logs:\n level: info\n traces:\n level: none\n\nextensions:\n healthcheckv2:\n use_v2: true\n http:\n endpoint: \"0.0.0.0:16687\"\n status:\n enabled: true\n path: \"/\"\n\n jaeger_query:\n storage:\n traces: some_storage\n\n jaeger_storage:\n backends:\n some_storage:\n elasticsearch:\n server_urls:\n - http://opensearch:9200\n service_cache_ttl: 1s\n\nreceivers:\n otlp:\n protocols:\n grpc:\n endpoint: \"0.0.0.0:4317\"\n http:\n endpoint: \"0.0.0.0:4318\"\n\n jaeger:\n protocols:\n grpc:\n thrift_binary:\n thrift_compact:\n thrift_http:\n endpoint: \"0.0.0.0:14268\"\n\n zipkin:\n endpoint: \"0.0.0.0:9411\"\n\nprocessors:\n filter/jaeger:\n error_mode: ignore\n traces:\n span:\n - 'resource.attributes[\"service.name\"] == \"jaeger\"'\n\n batch:\n\nexporters:\n jaeger_storage_exporter:\n trace_storage: some_storage\n" }, { "path": "jaeger-spark-dependencies-test/pom.xml", "content": "\n\n\n 4.0.0\n \n jaeger-spark-dependencies-parent\n io.jaegertracing.dependencies\n 0.0.1-SNAPSHOT\n \n\n jaeger-spark-dependencies-test\n\n \n \n junit\n junit\n ${version.junit}\n \n \n org.testcontainers\n testcontainers\n \n \n io.opentelemetry\n opentelemetry-api\n ${version.io.opentelemetry}\n \n \n io.opentelemetry\n opentelemetry-sdk\n ${version.io.opentelemetry}\n \n \n io.opentelemetry\n opentelemetry-exporter-otlp\n ${version.io.opentelemetry}\n \n \n io.opentelemetry.semconv\n opentelemetry-semconv\n 1.37.0\n \n \n io.opentracing\n opentracing-mock\n ${version.io.opentracing}\n test\n \n \n com.squareup.okhttp3\n okhttp\n ${version.com.squareup.okhttp3-okhttp}\n \n \n org.awaitility\n awaitility\n ${version.org.awaitility-awaitility}\n \n \n\n" }, { "path": "jaeger-spark-dependencies-test/src/main/java/io/jaegertracing/spark/dependencies/LogToConsolePrinter.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies;\n\nimport org.testcontainers.containers.output.OutputFrame;\n\nimport java.util.function.Consumer;\n\npublic final class LogToConsolePrinter implements Consumer {\n private final String prefix;\n\n public LogToConsolePrinter(String prefix) {\n this.prefix = prefix;\n }\n\n @Override\n public void accept(OutputFrame outputFrame) {\n String message = outputFrame.getUtf8String();\n if (message != null && !message.isEmpty()) {\n System.out.print(prefix);\n System.out.print(message);\n }\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/DependenciesTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test;\n\nimport static org.awaitility.Awaitility.await;\nimport static org.junit.Assert.assertEquals;\n\nimport com.fasterxml.jackson.core.type.TypeReference;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport io.opentelemetry.api.trace.Span;\nimport io.opentelemetry.api.trace.Tracer;\nimport io.jaegertracing.spark.dependencies.test.TracersGenerator.Flushable;\nimport io.jaegertracing.spark.dependencies.test.TracersGenerator.Tuple;\nimport io.jaegertracing.spark.dependencies.test.rest.DependencyLink;\nimport io.jaegertracing.spark.dependencies.test.rest.JsonHelper;\nimport io.jaegertracing.spark.dependencies.test.rest.RestResult;\nimport io.jaegertracing.spark.dependencies.test.tree.Node;\nimport io.jaegertracing.spark.dependencies.test.tree.TracingWrapper.OpenTelemetryWrapper;\nimport io.jaegertracing.spark.dependencies.test.tree.Traversals;\nimport io.jaegertracing.spark.dependencies.test.tree.TreeGenerator;\nimport java.io.IOException;\nimport java.util.HashMap;\nimport java.util.LinkedHashMap;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\nimport okhttp3.OkHttpClient;\nimport okhttp3.Request;\nimport okhttp3.Response;\nimport org.junit.Test;\n\n/**\n * @author Pavol Loffay\n */\npublic abstract class DependenciesTest {\n\n protected OkHttpClient okHttpClient = new OkHttpClient.Builder().build();\n protected ObjectMapper objectMapper = JsonHelper.configure(new ObjectMapper());\n\n /**\n * Set these in subclasses\n */\n protected static String queryUrl;\n protected static String collectorUrl;\n\n public static String jaegerVersion() {\n String jaegerVersion = System.getProperty(\"jaeger.version\", System.getenv(\"JAEGER_VERSION\"));\n return jaegerVersion != null ? jaegerVersion : \"latest\";\n }\n\n /**\n * Override this and run spark job\n */\n protected abstract void deriveDependencies() throws Exception;\n\n /**\n * Wait between submitting different traces\n */\n protected abstract void waitBetweenTraces() throws InterruptedException;\n\n @Test\n public void testJaegerOneTrace() throws Exception {\n System.out.println(\"=== Starting testJaegerOneTrace ===\");\n System.out.println(\"Generating Jaeger trace tree with 5 tracers, 50 nodes, depth 3...\");\n TreeGenerator treeGenerator = new TreeGenerator(\n TracersGenerator.generateJaeger(5, collectorUrl));\n Node root = treeGenerator.generateTree(50, 3);\n System.out.println(\"Trace tree generated. Root service: \" + root.getServiceName() + \", operation: \" + root.getTracingWrapper().operationName());\n \n System.out.println(\"Finishing spans...\");\n Traversals.postOrder(root, (node, parent) -> node.getTracingWrapper().get().getSpan().end());\n waitBetweenTraces();\n \n System.out.println(\"Flushing and closing tracers...\");\n treeGenerator.getTracers().forEach(tracer -> {\n tracer.flushable().flush();\n });\n \n // Give extra time for spans to be exported and indexed\n TimeUnit.SECONDS.sleep(2);\n \n System.out.println(\"Waiting for traces to appear in Jaeger Query...\");\n waitJaegerQueryContains(root.getServiceName(), root.getTracingWrapper().operationName());\n System.out.println(\"Traces found in Jaeger Query\");\n\n System.out.println(\"Deriving dependencies...\");\n deriveDependencies();\n System.out.println(\"Dependencies derived, asserting results...\");\n assertDependencies(DependencyLinkDerivator.serviceDependencies(root));\n System.out.println(\"=== testJaegerOneTrace completed successfully ===\");\n }\n\n @Test\n public void testJaegerMultipleTraces() throws Exception {\n System.out.println(\"=== Starting testJaegerMultipleTraces ===\");\n System.out.println(\"Generating 20 Jaeger trace trees with 50 tracers each...\");\n TreeGenerator treeGenerator = new TreeGenerator(\n TracersGenerator.generateJaeger(50, collectorUrl));\n Map> expectedDependencies = new LinkedHashMap<>();\n for (int i = 0; i < 20; i++) {\n System.out.println(\"Generating trace \" + (i + 1) + \"/20...\");\n Node root = treeGenerator.generateTree(50, 15);\n DependencyLinkDerivator.serviceDependencies(root, expectedDependencies);\n Traversals.postOrder(root, (node, parent) -> node.getTracingWrapper().get().getSpan().end());\n waitBetweenTraces();\n waitJaegerQueryContains(root.getServiceName(), root.getTracingWrapper().operationName());\n }\n System.out.println(\"All 20 traces generated and verified\");\n \n System.out.println(\"Flushing and closing tracers...\");\n // flush and wait for reported data\n treeGenerator.getTracers().forEach(tracer -> tracer.flushable().flush());\n \n // Give extra time for spans to be exported and indexed\n TimeUnit.SECONDS.sleep(2);\n\n System.out.println(\"Deriving dependencies...\");\n deriveDependencies();\n System.out.println(\"Dependencies derived, asserting results...\");\n assertDependencies(expectedDependencies);\n System.out.println(\"=== testJaegerMultipleTraces completed successfully ===\");\n }\n\n\n\n @Test\n public void testMultipleReferences() throws Exception {\n System.out.println(\"=== Starting testMultipleReferences ===\");\n System.out.println(\"Creating tracers for services S1, S2, S3...\");\n Tuple s1Tuple = TracersGenerator.createJaeger(\"S1\", collectorUrl);\n Tuple s2Tuple = TracersGenerator.createJaeger(\"S2\", collectorUrl);\n Tuple s3Tuple = TracersGenerator.createJaeger(\"S3\", collectorUrl);\n\n System.out.println(\"Creating spans with multiple references...\");\n // Note: OpenTelemetry doesn't support FOLLOWS_FROM references like OpenTracing did.\n // In OpenTelemetry, a span can only have one parent. Both s2Span and s3Span will have\n // s1Span as their parent, creating S1->S2 and S1->S3 dependencies.\n Span s1Span = s1Tuple.getA().spanBuilder(\"foo\").startSpan();\n Span s2Span = s2Tuple.getA().spanBuilder(\"bar\")\n .setParent(io.opentelemetry.context.Context.current().with(s1Span))\n .startSpan();\n Span s3Span = s3Tuple.getA().spanBuilder(\"baz\")\n .setParent(io.opentelemetry.context.Context.current().with(s1Span))\n .startSpan();\n\n System.out.println(\"Finishing and flushing spans...\");\n s1Span.end();\n s2Span.end();\n s3Span.end();\n s1Tuple.getB().flush();\n s2Tuple.getB().flush();\n s3Tuple.getB().flush();\n \n // Give extra time for spans to be exported and indexed\n TimeUnit.SECONDS.sleep(2);\n \n System.out.println(\"Waiting for traces to appear in Jaeger Query...\");\n waitJaegerQueryContains(\"S1\", \"foo\");\n waitJaegerQueryContains(\"S2\", \"bar\");\n waitJaegerQueryContains(\"S3\", \"baz\");\n System.out.println(\"All traces found in Jaeger Query\");\n\n System.out.println(\"Deriving dependencies...\");\n deriveDependencies();\n\n Map> expectedDependencies = new HashMap<>();\n Map s1Descendants = new HashMap<>();\n s1Descendants.put(\"S2\", 1L);\n s1Descendants.put(\"S3\", 1L);\n expectedDependencies.put(\"S1\", s1Descendants);\n System.out.println(\"Dependencies derived, asserting results...\");\n assertDependencies(expectedDependencies);\n System.out.println(\"=== testMultipleReferences completed successfully ===\");\n }\n\n protected void assertDependencies(Map> expectedDependencies) throws IOException {\n Request request = new Request.Builder()\n .url(queryUrl + \"/api/dependencies?endTs=\" + System.currentTimeMillis())\n .get()\n .build();\n try (Response response = okHttpClient.newCall(request).execute()) {\n assertEquals(200, response.code());\n RestResult restResult = objectMapper.readValue(response.body().string(), new TypeReference>() {});\n assertEquals(null, restResult.getErrors());\n assertEquals(expectedDependencies, DependencyLinkDerivator.serviceDependencies(restResult.getData()));\n }\n }\n\n protected void waitJaegerQueryContains(String service, String spanContainsThis) {\n String url = String.format(\"%s/api/traces?service=%s\", queryUrl, service);\n System.out.println(\"Waiting for trace in Jaeger Query. Service: \" + service + \", looking for: \" + spanContainsThis);\n System.out.println(\"Query URL: \" + url);\n \n Request request = new Request.Builder()\n .url(url)\n .get()\n .build();\n await()\n .pollInterval(1, TimeUnit.SECONDS)\n .atMost(30, TimeUnit.SECONDS)\n .until(() -> {\n try(Response response = okHttpClient.newCall(request).execute()) {\n String responseBody = response.body().string();\n int statusCode = response.code();\n boolean contains = responseBody.contains(spanContainsThis);\n \n if (!contains) {\n // Log the response when condition is not met to help with debugging\n System.out.println(\"Trace not found yet. Status code: \" + statusCode);\n System.out.println(\"Response body preview (first 500 chars): \" + \n (responseBody.length() > 500 ? responseBody.substring(0, 500) + \"...\" : responseBody));\n }\n \n return contains;\n }\n });\n System.out.println(\"Trace found for service: \" + service);\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/DependencyLinkDerivator.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test;\n\nimport io.jaegertracing.spark.dependencies.test.rest.DependencyLink;\nimport io.jaegertracing.spark.dependencies.test.tree.Node;\nimport io.jaegertracing.spark.dependencies.test.tree.TracingWrapper;\nimport io.jaegertracing.spark.dependencies.test.tree.Traversals;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * @author Pavol Loffay\n */\npublic class DependencyLinkDerivator {\n\n public static Map> serviceDependencies(Node root) {\n return serviceDependencies(root, new LinkedHashMap<>());\n }\n\n public static Map> serviceDependencies(Node root,\n Map> dependenciesMap) {\n\n Traversals.postOrder(root, (Node child, Node parent) -> {\n if (parent != null) {\n Map childMap = dependenciesMap.get(parent.getServiceName());\n if (childMap == null) {\n childMap = new LinkedHashMap<>();\n dependenciesMap.put(parent.getServiceName(), childMap);\n }\n\n Long callCount = childMap.get(child.getServiceName());\n if (callCount == null) {\n callCount = 0L;\n }\n childMap.put(child.getServiceName(), ++callCount);\n }\n });\n return dependenciesMap;\n }\n\n public static Map> serviceDependencies(List dependencyLinks) {\n Map> parentDependencyMap = new LinkedHashMap<>();\n dependencyLinks.forEach(dependencyLink -> {\n Map childCallCountMap = parentDependencyMap.get(dependencyLink.getParent());\n if (childCallCountMap == null) {\n childCallCountMap = new LinkedHashMap<>();\n parentDependencyMap.put(dependencyLink.getParent(), childCallCountMap);\n }\n childCallCountMap.put(dependencyLink.getChild(), dependencyLink.getCallCount());\n });\n return parentDependencyMap;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/TracersGenerator.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test;\n\nimport io.opentelemetry.api.OpenTelemetry;\nimport io.opentelemetry.api.trace.Tracer;\nimport io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter;\nimport io.opentelemetry.sdk.OpenTelemetrySdk;\nimport io.opentelemetry.sdk.resources.Resource;\nimport io.opentelemetry.sdk.trace.SdkTracerProvider;\nimport io.opentelemetry.sdk.trace.export.BatchSpanProcessor;\nimport io.opentelemetry.semconv.ServiceAttributes;\nimport io.jaegertracing.spark.dependencies.test.tree.TracingWrapper;\nimport io.jaegertracing.spark.dependencies.test.tree.TracingWrapper.OpenTelemetryWrapper;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.UUID;\nimport java.util.concurrent.TimeUnit;\n\n/**\n * @author Pavol Loffay\n */\npublic class TracersGenerator {\n\n public static class Tuple {\n private final A a;\n private final B b;\n\n Tuple(A a, B b) {\n this.a = a;\n this.b = b;\n }\n\n public A getA() {\n return a;\n }\n\n public B getB() {\n return b;\n }\n }\n\n public interface Flushable {\n void flush();\n }\n\n public static class TracerHolder {\n private T tracer;\n private String serviceName;\n private Flushable flushable;\n\n TracerHolder(T tracer, String serviceName, Flushable flushable) {\n this.tracer = tracer;\n this.serviceName = serviceName;\n this.flushable = flushable;\n }\n\n public T getTracer() {\n return tracer;\n }\n\n public TracingWrapper tracingWrapper() {\n return new OpenTelemetryWrapper((Tracer)tracer, serviceName);\n }\n\n public Flushable flushable() {\n return flushable;\n }\n }\n\n public static List> generateJaeger(int number, String collectorUrl) {\n List> tracers = new ArrayList<>(number);\n for (int i = 0; i < number; i++) {\n String serviceName = serviceName();\n Tuple jaegerTracer = createJaeger(serviceName, collectorUrl);\n tracers.add(new TracerHolder<>(jaegerTracer.getA(), serviceName, jaegerTracer.getB()));\n }\n return tracers;\n }\n\n public static Tuple createJaeger(String serviceName, String collectorUrl) {\n // Parse collectorUrl to extract host and port for OTLP gRPC\n // collectorUrl is in format \"http://host:port\"\n String host = \"localhost\";\n int port = 4317; // default\n \n try {\n // Parse the URL to extract host and port\n String urlStr = collectorUrl;\n // Remove scheme\n if (urlStr.startsWith(\"http://\")) {\n urlStr = urlStr.substring(7);\n } else if (urlStr.startsWith(\"https://\")) {\n urlStr = urlStr.substring(8);\n }\n // Remove path if present\n int slashIndex = urlStr.indexOf('/');\n if (slashIndex > 0) {\n urlStr = urlStr.substring(0, slashIndex);\n }\n // Extract host and port\n int colonIndex = urlStr.lastIndexOf(':');\n if (colonIndex > 0) {\n host = urlStr.substring(0, colonIndex);\n port = Integer.parseInt(urlStr.substring(colonIndex + 1));\n } else {\n host = urlStr;\n }\n } catch (Exception e) {\n System.err.println(\"[ERROR TracersGenerator] Failed to parse collectorUrl: \" + collectorUrl + \", error: \" + e.getMessage());\n }\n \n // Reconstruct endpoint in the format expected by gRPC exporter\n String otlpEndpoint = \"http://\" + host + \":\" + port;\n\n Resource resource = Resource.getDefault()\n .merge(Resource.builder()\n .put(ServiceAttributes.SERVICE_NAME, serviceName)\n .build());\n\n // For gRPC, the endpoint should include the scheme (http:// or https://)\n OtlpGrpcSpanExporter spanExporter = OtlpGrpcSpanExporter.builder()\n .setEndpoint(otlpEndpoint)\n .setTimeout(10, TimeUnit.SECONDS)\n .build();\n\n SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder()\n .addSpanProcessor(BatchSpanProcessor.builder(spanExporter)\n .setMaxQueueSize(100000)\n .setScheduleDelay(1, TimeUnit.MILLISECONDS)\n .build())\n .setResource(resource)\n .build();\n\n OpenTelemetry openTelemetry = OpenTelemetrySdk.builder()\n .setTracerProvider(sdkTracerProvider)\n .build();\n\n Tracer tracer = openTelemetry.getTracer(serviceName);\n\n return new Tuple<>(tracer,\n () -> {\n try {\n // Force flush to ensure all spans are exported\n sdkTracerProvider.forceFlush().join(10, TimeUnit.SECONDS);\n // Shutdown to ensure proper cleanup\n sdkTracerProvider.shutdown().join(10, TimeUnit.SECONDS);\n } catch (Exception ex) {\n throw new IllegalStateException(\"Failed to flush and shutdown tracer provider\", ex);\n }\n });\n }\n\n private static String serviceName() {\n return UUID.randomUUID().toString().replace(\"-\", \"\");\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/rest/DependencyLink.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test.rest;\n\n/**\n * @author Pavol Loffay\n */\npublic class DependencyLink {\n private String parent;\n private String child;\n private long callCount;\n\n // for jackson\n public DependencyLink() {}\n\n public DependencyLink(String parent, String child, long callCount) {\n this.parent = parent;\n this.child = child;\n this.callCount = callCount;\n }\n\n public String getParent() {\n return parent;\n }\n\n public void setParent(String parent) {\n this.parent = parent;\n }\n\n public String getChild() {\n return child;\n }\n\n public void setChild(String child) {\n this.child = child;\n }\n\n public long getCallCount() {\n return callCount;\n }\n\n public void setCallCount(long callCount) {\n this.callCount = callCount;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/rest/JsonHelper.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test.rest;\n\n\nimport com.fasterxml.jackson.databind.DeserializationFeature;\nimport com.fasterxml.jackson.databind.ObjectMapper;\n\n/**\n * @author Pavol Loffay\n */\npublic class JsonHelper {\n private JsonHelper() {}\n\n public static ObjectMapper configure(ObjectMapper objectMapper) {\n objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);\n return objectMapper;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/rest/RestResult.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test.rest;\n\nimport java.util.List;\n\n/**\n * @author Pavol Loffay\n */\npublic class RestResult {\n\n private List data;\n private List errors;\n\n public List getData() {\n return data;\n }\n\n public void setData(List data) {\n this.data = data;\n }\n\n public List getErrors() {\n return errors;\n }\n\n public void setErrors(List errors) {\n this.errors = errors;\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/tree/Node.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test.tree;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\n\n/**\n * Node in tree with N descendants. Node encapsulates\n * {@link TracingWrapper} which holds tracing information e.g span/tracer.\n *\n * @author Pavol Loffay\n */\npublic class Node {\n private List> descendants = new ArrayList<>();\n\n private TracingWrapper tracingWrapper;\n\n public Node(TracingWrapper tracingWrapper, Node parent) {\n this.tracingWrapper = tracingWrapper;\n\n if (parent != null) {\n tracingWrapper.createChildSpan(parent.getTracingWrapper());\n parent.addDescendant(this);\n } else {\n tracingWrapper.createChildSpan(null);\n }\n }\n\n public TracingWrapper getTracingWrapper() {\n return tracingWrapper;\n }\n\n private void addDescendant(Node descendant) {\n this.descendants.add(descendant);\n }\n\n public List getDescendants() {\n return Collections.unmodifiableList(descendants);\n }\n\n public String getServiceName() {\n return tracingWrapper.serviceName();\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/tree/TracingWrapper.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test.tree;\n\nimport io.opentelemetry.api.trace.Span;\nimport io.opentelemetry.api.trace.Tracer;\nimport io.opentelemetry.context.Context;\nimport java.util.UUID;\n\n/**\n * Encapsulates tracing information about one node(service) in the graph.\n * It allows to create one span for this node. Caller is responsible to\n * call {@link #createChildSpan(TracingWrapper)} and finish the span. The\n * parent parameter in createChildSpan should be of the same type as the\n * implementing wrapper.\n *\n * @author Pavol Loffay\n */\npublic interface TracingWrapper {\n T get();\n String serviceName();\n String operationName();\n void createChildSpan(TracingWrapper parent);\n\n class OpenTelemetryWrapper implements TracingWrapper {\n private final Tracer tracer;\n private final String serviceName;\n private Span span;\n private String operationName;\n\n public OpenTelemetryWrapper(Tracer tracer, String serviceName) {\n this.tracer = tracer;\n this.serviceName = serviceName;\n }\n\n @Override\n public OpenTelemetryWrapper get() {\n return this;\n }\n\n @Override\n public String serviceName() {\n return serviceName;\n }\n\n @Override\n public String operationName() {\n return operationName;\n }\n\n @Override\n public void createChildSpan(TracingWrapper parent) {\n operationName = UUID.randomUUID().toString().replace(\"-\", \"\");\n \n if (parent != null && parent.get().span != null) {\n Context parentContext = Context.current().with(parent.get().span);\n span = tracer.spanBuilder(operationName)\n .setParent(parentContext)\n .startSpan();\n } else {\n span = tracer.spanBuilder(operationName)\n .setNoParent()\n .startSpan();\n }\n }\n\n public Span getSpan() {\n return span;\n }\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/tree/Traversals.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test.tree;\n\nimport java.util.function.BiConsumer;\n\n/**\n * @author Pavol Loffay\n */\npublic class Traversals {\n\n /**\n * Traverse tree postOrder\n *\n * @param root root node\n * @param fce \n */\n public static void postOrder(Node root, BiConsumer, Node> fce) {\n postOrder(null, root, fce);\n }\n\n /**\n * @param node node\n * @param fce \n */\n private static void postOrder(Node parent, Node node, BiConsumer, Node> fce) {\n for (Node descendant : node.getDescendants()) {\n postOrder(node, descendant, fce);\n }\n fce.accept(node, parent);\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/main/java/io/jaegertracing/spark/dependencies/test/tree/TreeGenerator.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test.tree;\n\nimport io.jaegertracing.spark.dependencies.test.TracersGenerator.TracerHolder;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Queue;\nimport java.util.Random;\n\n/**\n * @author Pavol Loffay\n */\npublic class TreeGenerator {\n\n private Random tracersRandom = new Random();\n private List> tracers;\n\n public TreeGenerator(List> tracers) {\n if (tracers == null || tracers.isEmpty()) {\n throw new IllegalArgumentException();\n }\n this.tracers = new ArrayList<>(tracers);\n }\n\n public Node generateTree(int numOfNodes, int maxNumberOfDescendants) {\n if (numOfNodes <= 0 || maxNumberOfDescendants == 0) {\n throw new IllegalArgumentException();\n }\n\n Node root = new Node(tracers.get(0).tracingWrapper(), null);\n generateDescendants(new LinkedList<>(Collections.singletonList(root)), numOfNodes - 1, maxNumberOfDescendants);\n return root;\n }\n\n private void generateDescendants(Queue queue, int numOfNodes, final int maxNumberOfDescendants) {\n if (numOfNodes <= 0) {\n return;\n }\n\n Node parent = queue.poll();\n if (parent == null) {\n return;\n }\n for (int i = 0; i < maxNumberOfDescendants; i++) {\n Node descendant = new Node(tracers.get(tracersRandom.nextInt(tracers.size())).tracingWrapper(), parent);\n queue.add(descendant);\n if (--numOfNodes <= 0) {\n return;\n }\n }\n generateDescendants(queue, numOfNodes, maxNumberOfDescendants);\n }\n\n public List> getTracers() {\n return Collections.unmodifiableList(tracers);\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/test/java/io/jaegertracing/spark/dependencies/test/DependencyLinksDerivatorTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test;\n\nimport static org.junit.Assert.assertEquals;\n\nimport io.jaegertracing.spark.dependencies.test.rest.DependencyLink;\nimport io.jaegertracing.spark.dependencies.test.tree.Node;\nimport io.opentracing.mock.MockTracer;\nimport java.util.Arrays;\nimport java.util.Map;\nimport org.junit.Test;\n\n/**\n * @author Pavol Loffay\n */\npublic class DependencyLinksDerivatorTest {\n\n @Test\n public void testRootToMap() {\n Node root = new Node<>(new MockTracingWrapper(new MockTracer(), \"foo\"), null);\n new Node<>(new MockTracingWrapper(new MockTracer(), \"child1\"), root);\n new Node<>(new MockTracingWrapper(new MockTracer(), \"child1\"), root);\n new Node<>(new MockTracingWrapper(new MockTracer(), \"child2\"), root);\n Node child3 = new Node<>(new MockTracingWrapper(new MockTracer(), \"child3\"), root);\n Node child33 = new Node<>(new MockTracingWrapper(new MockTracer(), \"child33\"), child3);\n new Node<>(new MockTracingWrapper(new MockTracer(), \"child333\"), child33);\n\n Map> depLinks = DependencyLinkDerivator.serviceDependencies(root);\n // 3 parents\n assertEquals(3, depLinks.size());\n assertEquals(3, depLinks.get(\"foo\").size());\n assertEquals(1, depLinks.get(\"child3\").size());\n assertEquals(1, depLinks.get(\"child33\").size());\n\n assertEquals(Long.valueOf(2), depLinks.get(\"foo\").get(\"child1\"));\n assertEquals(Long.valueOf(1), depLinks.get(\"foo\").get(\"child2\"));\n assertEquals(Long.valueOf(1), depLinks.get(\"foo\").get(\"child3\"));\n assertEquals(Long.valueOf(1), depLinks.get(\"child3\").get(\"child33\"));\n assertEquals(Long.valueOf(1), depLinks.get(\"child33\").get(\"child333\"));\n }\n\n @Test\n public void testDepLinkToMap() {\n DependencyLink rootChild = new DependencyLink(\"root\", \"child\", 3);\n DependencyLink childRoot = new DependencyLink(\"child\", \"root\", 2);\n DependencyLink childChild2 = new DependencyLink(\"child\", \"child2\", 6);\n\n Map> depLinks = DependencyLinkDerivator.serviceDependencies(\n Arrays.asList(rootChild, childRoot, childChild2));\n\n assertEquals(2, depLinks.size());\n assertEquals(1, depLinks.get(\"root\").size());\n assertEquals(2, depLinks.get(\"child\").size());\n\n assertEquals(Long.valueOf(3), depLinks.get(\"root\").get(\"child\"));\n assertEquals(Long.valueOf(2), depLinks.get(\"child\").get(\"root\"));\n assertEquals(Long.valueOf(6), depLinks.get(\"child\").get(\"child2\"));\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/test/java/io/jaegertracing/spark/dependencies/test/MockTracingWrapper.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test;\n\nimport io.jaegertracing.spark.dependencies.test.tree.TracingWrapper;\nimport io.opentracing.mock.MockSpan;\nimport io.opentracing.mock.MockTracer;\nimport io.opentracing.mock.MockTracer.SpanBuilder;\n\n/**\n * @author Pavol Loffay\n */\npublic class MockTracingWrapper implements TracingWrapper {\n\n private String serviceName;\n private MockTracer tracer;\n private MockSpan span;\n\n public MockTracingWrapper(MockTracer mockTracer, String serviceName) {\n this.serviceName = serviceName;\n this.tracer = mockTracer;\n }\n\n @Override\n public MockTracingWrapper get() {\n return this;\n }\n\n @Override\n public String serviceName() {\n return serviceName;\n }\n\n @Override\n public String operationName() {\n return span != null ? span.operationName() : null;\n }\n\n @Override\n public void createChildSpan(TracingWrapper parent) {\n SpanBuilder spanBuilder = tracer.buildSpan(parent == null ? \"|\" : parent.get().operationName() + \"->\");\n if (parent != null) {\n spanBuilder.asChildOf(parent.get().span);\n }\n span = spanBuilder.start();\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/test/java/io/jaegertracing/spark/dependencies/test/rest/DeserializationTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test.rest;\n\n\nimport com.fasterxml.jackson.core.type.TypeReference;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport java.io.IOException;\nimport org.junit.Assert;\nimport org.junit.Test;\n\n/**\n * @author Pavol Loffay\n */\npublic class DeserializationTest {\n private ObjectMapper objectMapper = JsonHelper.configure(new ObjectMapper());\n\n @Test\n public void testDependencyLinkDeserialization() throws IOException {\n String json = \"{\\\"data\\\":[{\\\"parent\\\":\\\"service1\\\",\\\"child\\\":\\\"service2\\\",\\\"callCount\\\":1}],\\\"total\\\":0,\" +\n \"\\\"limit\\\":0,\\\"offset\\\":0,\\\"errors\\\":null}\";\n\n RestResult restResult = objectMapper.readValue(json, new TypeReference>() {});\n Assert.assertEquals(null, restResult.getErrors());\n Assert.assertEquals(1, restResult.getData().size());\n Assert.assertEquals(\"service1\", restResult.getData().get(0).getParent());\n Assert.assertEquals(\"service2\", restResult.getData().get(0).getChild());\n Assert.assertEquals(1L, restResult.getData().get(0).getCallCount());\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/test/java/io/jaegertracing/spark/dependencies/test/tree/TraversalsTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test.tree;\n\nimport static org.junit.Assert.assertEquals;\n\nimport io.jaegertracing.spark.dependencies.test.MockTracingWrapper;\nimport io.opentracing.mock.MockTracer;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport org.junit.Test;\n\n/**\n * @author Pavol Loffay\n */\npublic class TraversalsTest {\n\n @Test\n public void testInorder() {\n Node root = new Node<>(new MockTracingWrapper(new MockTracer(), \"foo\"), null);\n Node child1 = new Node<>(new MockTracingWrapper(new MockTracer(), \"child1\"), root);\n Node child2 = new Node<>(new MockTracingWrapper(new MockTracer(), \"child2\"), root);\n Node child3 = new Node<>(new MockTracingWrapper(new MockTracer(), \"child3\"), root);\n\n List nodes = new ArrayList<>();\n Traversals.postOrder(root, (node, parent) -> {\n if (parent != null) {\n assertEquals(root, parent);\n } else {\n assertEquals(null, parent);\n }\n nodes.add(node);\n });\n assertEquals(new ArrayList<>(Arrays.asList(child1, child2, child3, root)), nodes);\n\n Node child33 = new Node<>(new MockTracingWrapper(new MockTracer(), \"child33\"), child3);\n Node child333 = new Node<>(new MockTracingWrapper(new MockTracer(), \"child333\"), child33);\n\n List nodes2 = new ArrayList<>();\n List parents2 = new ArrayList<>();\n Traversals.postOrder(root, (node, parent) -> {\n nodes2.add(node);\n parents2.add(parent);\n });\n assertEquals(new ArrayList<>(Arrays.asList(child1, child2, child333, child33, child3, root)), nodes2);\n assertEquals(new ArrayList<>(Arrays.asList(root, root, child33, child3, root, null)), parents2);\n }\n}\n" }, { "path": "jaeger-spark-dependencies-test/src/test/java/io/jaegertracing/spark/dependencies/test/tree/TreeGeneratorTest.java", "content": "/**\n * Copyright (c) The Jaeger Authors\n * SPDX-License-Identifier: Apache-2.0\n */\npackage io.jaegertracing.spark.dependencies.test.tree;\n\nimport static junit.framework.TestCase.assertTrue;\nimport static org.junit.Assert.assertEquals;\nimport static org.junit.Assert.assertNotNull;\n\nimport io.jaegertracing.spark.dependencies.test.TracersGenerator;\nimport io.jaegertracing.spark.dependencies.test.tree.TracingWrapper.OpenTelemetryWrapper;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.junit.Test;\n\n/**\n * @author Pavol Loffay\n */\npublic class TreeGeneratorTest {\n\n @Test\n public void testGenerateOne() {\n Node root = new TreeGenerator(TracersGenerator.generateJaeger(1, \"http://localhost\"))\n .generateTree(1, 3);\n assertEquals(0, root.getDescendants().size());\n assertNotNull(root.getServiceName());\n assertNotNull(root.getTracingWrapper().get().getSpan());\n assertNotNull(root.getTracingWrapper().get().operationName());\n }\n\n @Test\n public void testBranchingFactorOne() {\n Node root = new TreeGenerator(TracersGenerator.generateJaeger(1, \"http://localhost\"))\n .generateTree(16, 3);\n List nodes = new ArrayList<>();\n Traversals.postOrder(root, (jaegerWrapperNode, jaegerWrapperNode2) -> {\n assertTrue(jaegerWrapperNode.getDescendants().size() <= 3);\n nodes.add(jaegerWrapperNode);\n });\n assertEquals(16, nodes.size());\n }\n}\n" }, { "path": "mvnw", "content": "#!/bin/sh\n# ----------------------------------------------------------------------------\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n# ----------------------------------------------------------------------------\n\n# ----------------------------------------------------------------------------\n# Maven2 Start Up Batch script\n#\n# Required ENV vars:\n# ------------------\n# JAVA_HOME - location of a JDK home dir\n#\n# Optional ENV vars\n# -----------------\n# M2_HOME - location of maven2's installed home dir\n# MAVEN_OPTS - parameters passed to the Java VM when running Maven\n# e.g. to debug Maven itself, use\n# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000\n# MAVEN_SKIP_RC - flag to disable loading of mavenrc files\n# ----------------------------------------------------------------------------\n\nif [ -z \"$MAVEN_SKIP_RC\" ] ; then\n\n if [ -f /etc/mavenrc ] ; then\n . /etc/mavenrc\n fi\n\n if [ -f \"$HOME/.mavenrc\" ] ; then\n . \"$HOME/.mavenrc\"\n fi\n\nfi\n\n# OS specific support. $var _must_ be set to either true or false.\ncygwin=false;\ndarwin=false;\nmingw=false\ncase \"`uname`\" in\n CYGWIN*) cygwin=true ;;\n MINGW*) mingw=true;;\n Darwin*) darwin=true\n #\n # Look for the Apple JDKs first to preserve the existing behaviour, and then look\n # for the new JDKs provided by Oracle.\n #\n if [ -z \"$JAVA_HOME\" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then\n #\n # Apple JDKs\n #\n export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home\n fi\n\n if [ -z \"$JAVA_HOME\" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then\n #\n # Apple JDKs\n #\n export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home\n fi\n\n if [ -z \"$JAVA_HOME\" ] && [ -L \"/Library/Java/JavaVirtualMachines/CurrentJDK\" ] ; then\n #\n # Oracle JDKs\n #\n export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home\n fi\n\n if [ -z \"$JAVA_HOME\" ] && [ -x \"/usr/libexec/java_home\" ]; then\n #\n # Apple JDKs\n #\n export JAVA_HOME=`/usr/libexec/java_home`\n fi\n ;;\nesac\n\nif [ -z \"$JAVA_HOME\" ] ; then\n if [ -r /etc/gentoo-release ] ; then\n JAVA_HOME=`java-config --jre-home`\n fi\nfi\n\nif [ -z \"$M2_HOME\" ] ; then\n ## resolve links - $0 may be a link to maven's home\n PRG=\"$0\"\n\n # need this for relative symlinks\n while [ -h \"$PRG\" ] ; do\n ls=`ls -ld \"$PRG\"`\n link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n if expr \"$link\" : '/.*' > /dev/null; then\n PRG=\"$link\"\n else\n PRG=\"`dirname \"$PRG\"`/$link\"\n fi\n done\n\n saveddir=`pwd`\n\n M2_HOME=`dirname \"$PRG\"`/..\n\n # make it fully qualified\n M2_HOME=`cd \"$M2_HOME\" && pwd`\n\n cd \"$saveddir\"\n # echo Using m2 at $M2_HOME\nfi\n\n# For Cygwin, ensure paths are in UNIX format before anything is touched\nif $cygwin ; then\n [ -n \"$M2_HOME\" ] &&\n M2_HOME=`cygpath --unix \"$M2_HOME\"`\n [ -n \"$JAVA_HOME\" ] &&\n JAVA_HOME=`cygpath --unix \"$JAVA_HOME\"`\n [ -n \"$CLASSPATH\" ] &&\n CLASSPATH=`cygpath --path --unix \"$CLASSPATH\"`\nfi\n\n# For Migwn, ensure paths are in UNIX format before anything is touched\nif $mingw ; then\n [ -n \"$M2_HOME\" ] &&\n M2_HOME=\"`(cd \"$M2_HOME\"; pwd)`\"\n [ -n \"$JAVA_HOME\" ] &&\n JAVA_HOME=\"`(cd \"$JAVA_HOME\"; pwd)`\"\n # TODO classpath?\nfi\n\nif [ -z \"$JAVA_HOME\" ]; then\n javaExecutable=\"`which javac`\"\n if [ -n \"$javaExecutable\" ] && ! [ \"`expr \\\"$javaExecutable\\\" : '\\([^ ]*\\)'`\" = \"no\" ]; then\n # readlink(1) is not available as standard on Solaris 10.\n readLink=`which readlink`\n if [ ! `expr \"$readLink\" : '\\([^ ]*\\)'` = \"no\" ]; then\n if $darwin ; then\n javaHome=\"`dirname \\\"$javaExecutable\\\"`\"\n javaExecutable=\"`cd \\\"$javaHome\\\" && pwd -P`/javac\"\n else\n javaExecutable=\"`readlink -f \\\"$javaExecutable\\\"`\"\n fi\n javaHome=\"`dirname \\\"$javaExecutable\\\"`\"\n javaHome=`expr \"$javaHome\" : '\\(.*\\)/bin'`\n JAVA_HOME=\"$javaHome\"\n export JAVA_HOME\n fi\n fi\nfi\n\nif [ -z \"$JAVACMD\" ] ; then\n if [ -n \"$JAVA_HOME\" ] ; then\n if [ -x \"$JAVA_HOME/jre/sh/java\" ] ; then\n # IBM's JDK on AIX uses strange locations for the executables\n JAVACMD=\"$JAVA_HOME/jre/sh/java\"\n else\n JAVACMD=\"$JAVA_HOME/bin/java\"\n fi\n else\n JAVACMD=\"`which java`\"\n fi\nfi\n\nif [ ! -x \"$JAVACMD\" ] ; then\n echo \"Error: JAVA_HOME is not defined correctly.\" >&2\n echo \" We cannot execute $JAVACMD\" >&2\n exit 1\nfi\n\nif [ -z \"$JAVA_HOME\" ] ; then\n echo \"Warning: JAVA_HOME environment variable is not set.\"\nfi\n\nCLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher\n\n# traverses directory structure from process work directory to filesystem root\n# first directory with .mvn subdirectory is considered project base directory\nfind_maven_basedir() {\n local basedir=$(pwd)\n local wdir=$(pwd)\n while [ \"$wdir\" != '/' ] ; do\n if [ -d \"$wdir\"/.mvn ] ; then\n basedir=$wdir\n break\n fi\n wdir=$(cd \"$wdir/..\"; pwd)\n done\n echo \"${basedir}\"\n}\n\n# concatenates all lines of a file\nconcat_lines() {\n if [ -f \"$1\" ]; then\n echo \"$(tr -s '\\n' ' ' < \"$1\")\"\n fi\n}\n\nexport MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)}\nMAVEN_OPTS=\"$(concat_lines \"$MAVEN_PROJECTBASEDIR/.mvn/jvm.config\") $MAVEN_OPTS\"\n\n# For Cygwin, switch paths to Windows format before running java\nif $cygwin; then\n [ -n \"$M2_HOME\" ] &&\n M2_HOME=`cygpath --path --windows \"$M2_HOME\"`\n [ -n \"$JAVA_HOME\" ] &&\n JAVA_HOME=`cygpath --path --windows \"$JAVA_HOME\"`\n [ -n \"$CLASSPATH\" ] &&\n CLASSPATH=`cygpath --path --windows \"$CLASSPATH\"`\n [ -n \"$MAVEN_PROJECTBASEDIR\" ] &&\n MAVEN_PROJECTBASEDIR=`cygpath --path --windows \"$MAVEN_PROJECTBASEDIR\"`\nfi\n\n# Provide a \"standardized\" way to retrieve the CLI args that will\n# work with both Windows and non-Windows executions.\nMAVEN_CMD_LINE_ARGS=\"$MAVEN_CONFIG $@\"\nexport MAVEN_CMD_LINE_ARGS\n\nWRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain\n\n# avoid using MAVEN_CMD_LINE_ARGS below since that would loose parameter escaping in $@\nexec \"$JAVACMD\" \\\n $MAVEN_OPTS \\\n -classpath \"$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar\" \\\n \"-Dmaven.home=${M2_HOME}\" \"-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}\" \\\n ${WRAPPER_LAUNCHER} $MAVEN_CONFIG \"$@\"\n" }, { "path": "mvnw.cmd", "content": "@REM ----------------------------------------------------------------------------\r\n@REM Licensed to the Apache Software Foundation (ASF) under one\r\n@REM or more contributor license agreements. See the NOTICE file\r\n@REM distributed with this work for additional information\r\n@REM regarding copyright ownership. The ASF licenses this file\r\n@REM to you under the Apache License, Version 2.0 (the\r\n@REM \"License\"); you may not use this file except in compliance\r\n@REM with the License. You may obtain a copy of the License at\r\n@REM\r\n@REM http://www.apache.org/licenses/LICENSE-2.0\r\n@REM\r\n@REM Unless required by applicable law or agreed to in writing,\r\n@REM software distributed under the License is distributed on an\r\n@REM \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n@REM KIND, either express or implied. See the License for the\r\n@REM specific language governing permissions and limitations\r\n@REM under the License.\r\n@REM ----------------------------------------------------------------------------\r\n\r\n@REM ----------------------------------------------------------------------------\r\n@REM Maven2 Start Up Batch script\r\n@REM\r\n@REM Required ENV vars:\r\n@REM JAVA_HOME - location of a JDK home dir\r\n@REM\r\n@REM Optional ENV vars\r\n@REM M2_HOME - location of maven2's installed home dir\r\n@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands\r\n@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending\r\n@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven\r\n@REM e.g. to debug Maven itself, use\r\n@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000\r\n@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files\r\n@REM ----------------------------------------------------------------------------\r\n\r\n@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'\r\n@echo off\r\n@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on'\r\n@if \"%MAVEN_BATCH_ECHO%\" == \"on\" echo %MAVEN_BATCH_ECHO%\r\n\r\n@REM set %HOME% to equivalent of $HOME\r\nif \"%HOME%\" == \"\" (set \"HOME=%HOMEDRIVE%%HOMEPATH%\")\r\n\r\n@REM Execute a user defined script before this one\r\nif not \"%MAVEN_SKIP_RC%\" == \"\" goto skipRcPre\r\n@REM check for pre script, once with legacy .bat ending and once with .cmd ending\r\nif exist \"%HOME%\\mavenrc_pre.bat\" call \"%HOME%\\mavenrc_pre.bat\"\r\nif exist \"%HOME%\\mavenrc_pre.cmd\" call \"%HOME%\\mavenrc_pre.cmd\"\r\n:skipRcPre\r\n\r\n@setlocal\r\n\r\nset ERROR_CODE=0\r\n\r\n@REM To isolate internal variables from possible post scripts, we use another setlocal\r\n@setlocal\r\n\r\n@REM ==== START VALIDATION ====\r\nif not \"%JAVA_HOME%\" == \"\" goto OkJHome\r\n\r\necho.\r\necho Error: JAVA_HOME not found in your environment. >&2\r\necho Please set the JAVA_HOME variable in your environment to match the >&2\r\necho location of your Java installation. >&2\r\necho.\r\ngoto error\r\n\r\n:OkJHome\r\nif exist \"%JAVA_HOME%\\bin\\java.exe\" goto init\r\n\r\necho.\r\necho Error: JAVA_HOME is set to an invalid directory. >&2\r\necho JAVA_HOME = \"%JAVA_HOME%\" >&2\r\necho Please set the JAVA_HOME variable in your environment to match the >&2\r\necho location of your Java installation. >&2\r\necho.\r\ngoto error\r\n\r\n@REM ==== END VALIDATION ====\r\n\r\n:init\r\n\r\nset MAVEN_CMD_LINE_ARGS=%MAVEN_CONFIG% %*\r\n\r\n@REM Find the project base dir, i.e. the directory that contains the folder \".mvn\".\r\n@REM Fallback to current working directory if not found.\r\n\r\nset MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%\r\nIF NOT \"%MAVEN_PROJECTBASEDIR%\"==\"\" goto endDetectBaseDir\r\n\r\nset EXEC_DIR=%CD%\r\nset WDIR=%EXEC_DIR%\r\n:findBaseDir\r\nIF EXIST \"%WDIR%\"\\.mvn goto baseDirFound\r\ncd ..\r\nIF \"%WDIR%\"==\"%CD%\" goto baseDirNotFound\r\nset WDIR=%CD%\r\ngoto findBaseDir\r\n\r\n:baseDirFound\r\nset MAVEN_PROJECTBASEDIR=%WDIR%\r\ncd \"%EXEC_DIR%\"\r\ngoto endDetectBaseDir\r\n\r\n:baseDirNotFound\r\nset MAVEN_PROJECTBASEDIR=%EXEC_DIR%\r\ncd \"%EXEC_DIR%\"\r\n\r\n:endDetectBaseDir\r\n\r\nIF NOT EXIST \"%MAVEN_PROJECTBASEDIR%\\.mvn\\jvm.config\" goto endReadAdditionalConfig\r\n\r\n@setlocal EnableExtensions EnableDelayedExpansion\r\nfor /F \"usebackq delims=\" %%a in (\"%MAVEN_PROJECTBASEDIR%\\.mvn\\jvm.config\") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a\r\n@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%\r\n\r\n:endReadAdditionalConfig\r\n\r\nSET MAVEN_JAVA_EXE=\"%JAVA_HOME%\\bin\\java.exe\"\r\n\nset WRAPPER_JAR=\"\"%MAVEN_PROJECTBASEDIR%\\.mvn\\wrapper\\maven-wrapper.jar\"\"\nset WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain\r\n\r\n# avoid using MAVEN_CMD_LINE_ARGS below since that would loose parameter escaping in %*\r\n%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% \"-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%\" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*\r\nif ERRORLEVEL 1 goto error\r\ngoto end\r\n\r\n:error\r\nset ERROR_CODE=1\r\n\r\n:end\r\n@endlocal & set ERROR_CODE=%ERROR_CODE%\r\n\r\nif not \"%MAVEN_SKIP_RC%\" == \"\" goto skipRcPost\r\n@REM check for post script, once with legacy .bat ending and once with .cmd ending\r\nif exist \"%HOME%\\mavenrc_post.bat\" call \"%HOME%\\mavenrc_post.bat\"\r\nif exist \"%HOME%\\mavenrc_post.cmd\" call \"%HOME%\\mavenrc_post.cmd\"\r\n:skipRcPost\r\n\r\n@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'\r\nif \"%MAVEN_BATCH_PAUSE%\" == \"on\" pause\r\n\r\nif \"%MAVEN_TERMINATE_CMD%\" == \"on\" exit %ERROR_CODE%\r\n\r\nexit /B %ERROR_CODE%\r\n" }, { "path": "pom.xml", "content": "\n\n\n 4.0.0\n\n io.jaegertracing.dependencies\n jaeger-spark-dependencies-parent\n 0.0.1-SNAPSHOT\n pom\n\n \n jaeger-spark-dependencies-test\n jaeger-spark-dependencies-common\n jaeger-spark-dependencies-cassandra\n jaeger-spark-dependencies-elasticsearch\n jaeger-spark-dependencies-opensearch\n jaeger-spark-dependencies\n \n\n \n 21\n 21\n UTF-8\n UTF-8\n\n 2.12\n 3.5.8\n 0.35.5\n 0.33.0\n 1.57.0\n 2.10.0\n 4.13.2\n 3.27.6\n 1.21.4\n 4.12.0\n 4.3.0\n\n 3.0\n 3.11.0\n 3.1.1\n 3.2.1\n 0.3.4\n 3.6.2\n 2.21.1\n 3.3.6\n 9.1.3\n \n\n 2017\n\n \n \n The Apache Software License, Version 2.0\n http://www.apache.org/licenses/LICENSE-2.0.txt\n repo\n \n \n\n \n \n jaegertracing\n Jaegertracing Gitter\n https://gitter.im/jaegertracing/Lobby\n \n \n\n \n \n \n ${project.groupId}\n jaeger-spark-dependencies-common\n ${project.version}\n \n \n ${project.groupId}\n jaeger-spark-dependencies-test\n ${project.version}\n \n \n ${project.groupId}\n jaeger-spark-dependencies-cassandra\n ${project.version}\n \n \n ${project.groupId}\n jaeger-spark-dependencies-elasticsearch\n ${project.version}\n \n \n ${project.groupId}\n jaeger-spark-dependencies-opensearch\n ${project.version}\n \n\n \n org.apache.spark\n spark-core_${version.scala.binary}\n ${version.org.apache.spark}\n \n \n \n commons-codec\n commons-codec\n \n \n org.apache.commons\n commons-compress\n \n \n org.apache.zookeeper\n zookeeper\n \n \n com.fasterxml.jackson.module\n jackson-module-scala_${version.scala.binary}\n \n \n org.apache.hadoop\n hadoop-client-api\n \n \n org.apache.hadoop\n hadoop-client-runtime\n \n \n \n\n \n org.testcontainers\n testcontainers\n ${version.org.testcontainers}\n \n \n org.testcontainers\n cassandra\n ${version.org.testcontainers}\n \n \n \n com.datastax.cassandra\n cassandra-driver-core\n \n \n \n \n \n com.squareup.okio\n okio\n 3.16.4\n \n \n com.squareup.okio\n okio-jvm\n 3.16.4\n \n \n com.google.guava\n guava\n 32.1.3-jre\n \n \n \n commons-collections\n commons-collections\n 3.2.2\n \n \n commons-beanutils\n commons-beanutils\n 1.11.0\n \n \n org.scala-lang\n scala-library\n ${version.scala.binary}.18\n \n \n \n\n \n \n org.apache.spark\n spark-core_${version.scala.binary}\n \n\n \n \n org.apache.spark\n spark-sql_${version.scala.binary}\n ${version.org.apache.spark}\n \n \n org.apache.avro\n avro\n \n \n com.fasterxml.jackson.module\n jackson-module-scala_${version.scala.binary}\n \n \n \n \n \n com.google.protobuf\n protobuf-java\n 3.25.8\n \n \n org.apache.zookeeper\n zookeeper\n 3.9.5\n \n \n \n com.fasterxml.jackson.core\n jackson-core\n ${version.jackson}\n \n \n com.fasterxml.jackson.core\n jackson-databind\n ${version.jackson}\n \n \n com.fasterxml.jackson.module\n jackson-module-scala_${version.scala.binary}\n ${version.jackson}\n \n \n com.fasterxml.jackson.jaxrs\n jackson-jaxrs-base\n ${version.jackson}\n \n \n com.fasterxml.jackson.jaxrs\n jackson-jaxrs-json-provider\n ${version.jackson}\n \n \n org.apache.hadoop\n hadoop-client\n ${version.hadoop.client}\n \n \n org.apache.hadoop.thirdparty\n hadoop-shaded-protobuf_3_7\n \n \n \n \n io.netty\n netty-all\n 4.2.9.Final\n \n \n org.apache.avro\n avro\n 1.12.1\n \n \n \n \n junit\n junit\n ${version.junit}\n test\n \n \n org.assertj\n assertj-core\n ${version.org.assertj}\n test\n \n \n\n \n \n \n \n com.mycila\n license-maven-plugin\n ${version.maven-license-plugin}\n \n \n \n io.takari\n maven\n ${version.maven-plugi}\n \n \n org.apache.maven.plugins\n maven-surefire-plugin\n 2.22.2\n \n \n \n\n \n \n com.mycila\n license-maven-plugin\n \n
header.txt
\n true\n true\n \n SCRIPT_STYLE\n SCRIPT_STYLE\n \n \n LICENSE\n mvnw\n mvnw.cmd\n .mvn/wrapper/maven-wrapper.properties\n **swp\n .github/java-upgrade/**\n \n **/ElasticsearchDependenciesJob.java\n **/OpenSearchDependenciesJob.java\n **/CassandraDependenciesJob.java\n \n \n \n \n \n check\n \n compile\n \n \n \n \n \n\n \n \n \n java17-plus\n \n [17,)\n \n \n \n \n org.apache.maven.plugins\n maven-surefire-plugin\n \n \n --add-opens=java.base/java.lang=ALL-UNNAMED\n --add-opens=java.base/java.lang.invoke=ALL-UNNAMED\n --add-opens=java.base/java.lang.reflect=ALL-UNNAMED\n --add-opens=java.base/java.io=ALL-UNNAMED\n --add-opens=java.base/java.net=ALL-UNNAMED\n --add-opens=java.base/java.nio=ALL-UNNAMED\n --add-opens=java.base/java.util=ALL-UNNAMED\n --add-opens=java.base/java.util.concurrent=ALL-UNNAMED\n --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED\n --add-opens=java.base/sun.nio.ch=ALL-UNNAMED\n --add-opens=java.base/sun.nio.cs=ALL-UNNAMED\n --add-opens=java.base/sun.security.action=ALL-UNNAMED\n --add-opens=java.base/sun.util.calendar=ALL-UNNAMED\n -Djdk.reflect.useDirectMethodHandle=false\n \n \n \n \n \n \n \n\n" }, { "path": "renovate.json", "content": "{\n \"$schema\": \"https://docs.renovatebot.com/renovate-schema.json\",\n \"extends\": [\n \"config:best-practices\",\n \":gitSignOff\"\n ],\n \"schedule\": [\n \"on the first day of the month\"\n ],\n \"dependencyDashboard\": true,\n \"labels\": [\n \"changelog:dependencies\"\n ],\n \"packageRules\": [\n {\n \"matchManagers\": [\"github-actions\"],\n \"matchUpdateTypes\": [\"major\", \"minor\"],\n \"groupName\": \"GitHub Actions\"\n },\n {\n \"matchManagers\": [\"github-actions\"],\n \"matchUpdateTypes\": [\"patch\"],\n \"enabled\": false\n }\n ]\n}\n" } ]