[ { "path": ".github/workflows/build.yml", "content": "name: Build\n\non:\n pull_request:\n\npermissions:\n contents: read\n\njobs:\n build:\n name: Build\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n\n - name: Setup Java\n uses: actions/setup-java@v4\n with:\n java-version: 21\n distribution: zulu\n cache: gradle\n\n - name: Build\n shell: bash\n run: ./gradlew build -S\n" }, { "path": ".github/workflows/cache.yml", "content": "name: Clear cache\n\non:\n schedule:\n - cron: '0 3 * * *'\n workflow_dispatch:\n\njobs:\n clear:\n name: Delete all caches\n runs-on: ubuntu-latest\n steps:\n - name: Clear caches\n uses: easimon/wipe-cache@v2\n" }, { "path": ".github/workflows/early-access.yml", "content": "name: EarlyAccess\n\non:\n push:\n branches: [ main ]\n\npermissions:\n actions: write\n id-token: write\n contents: write\n\njobs:\n precheck:\n name: Precheck\n uses: redis/riot/.github/workflows/step-precheck.yml@main\n secrets: inherit\n\n jlink:\n name: Jlink\n needs: [precheck]\n if: endsWith(${{ needs.precheck.outputs.version }}, '-SNAPSHOT')\n uses: redis/riot/.github/workflows/step-jlink.yml@main\n with:\n project-version: ${{ needs.precheck.outputs.version }}\n secrets:\n codecov-token: ${{ secrets.CODECOV_TOKEN }}\n\n release:\n name: Release\n needs: [precheck, jlink]\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n with:\n fetch-depth: 0\n\n - name: Download artifacts\n uses: actions/download-artifact@v4\n with:\n name: artifacts\n path: plugins/riot/build\n\n - name: Download repos\n uses: actions/download-artifact@v4\n with:\n name: repos\n path: build/repos/local/release\n\n - name: Download jlink\n uses: actions/download-artifact@v4\n with:\n name: jlink\n path: out/jreleaser/assemble/riot-standalone/jlink\n\n - name: Release\n uses: jreleaser/release-action@v2\n with:\n arguments: full-release\n env:\n JRELEASER_DOCKER_DEFAULT_USERNAME: ${{ secrets.DOCKER_USERNAME }}\n JRELEASER_DOCKER_DEFAULT_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}\n JRELEASER_GITHUB_PASSWORD: ${{ secrets.GIT_ACCESS_TOKEN }}\n JRELEASER_GITHUB_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}\n JRELEASER_GITHUB_USERNAME: ${{ secrets.GIT_USER }}\n JRELEASER_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}\n JRELEASER_GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }}\n JRELEASER_GPG_SECRET_KEY: ${{ secrets.GPG_SECRET_KEY }}\n JRELEASER_NEXUS2_USERNAME: ${{ secrets.SONATYPE_USERNAME }}\n JRELEASER_NEXUS2_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}\n JRELEASER_PROJECT_VERSION: ${{ needs.precheck.outputs.version }}\n JRELEASER_SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}\n\n - name: JReleaser release output\n if: always()\n uses: actions/upload-artifact@v4\n with:\n name: riot-release\n path: |\n out/jreleaser/trace.log\n out/jreleaser/output.properties" }, { "path": ".github/workflows/publish.yml", "content": "name: Publish\n\non:\n workflow_dispatch:\n inputs:\n version:\n description: 'Release version'\n required: true\n\npermissions:\n actions: write\n id-token: write\n contents: write\n\njobs:\n precheck:\n name: Precheck\n runs-on: ubuntu-latest\n outputs:\n version: ${{ steps.vars.outputs.VERSION }}\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n publish:\n name: Publish\n needs: [precheck]\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n with:\n ref: main\n fetch-depth: 0\n\n - name: Version\n id: vars\n shell: bash\n run: |\n echo \"VERSION=${{ github.event.inputs.version }}\" >> $GITHUB_OUTPUT\n echo ${{ github.event.inputs.version }} > VERSION\n\n - name: Setup Java\n uses: actions/setup-java@v4\n with:\n java-version: ${{ vars.JAVA_VERSION }}\n distribution: ${{ vars.JAVA_DISTRO }}\n cache: gradle\n\n - name: Deploy\n env:\n GRGIT_USER: ${{ secrets.GIT_USER }}\n GRGIT_PASS: ${{ secrets.GIT_ACCESS_TOKEN }}\n run: |\n ./gradlew -Prelease=true -PreproducibleBuild=true gitPublishPush -S" }, { "path": ".github/workflows/release.yml", "content": "name: Release\n\non:\n workflow_dispatch:\n inputs:\n version:\n description: 'Release version'\n required: true\n build-tasks:\n description: 'Build tasks'\n default: 'build aggregateTestReports aggregateJacocoReport publish'\n required: false\n type: string\n\npermissions:\n actions: write\n id-token: write\n contents: write\n\njobs:\n precheck:\n name: Precheck\n runs-on: ubuntu-latest\n outputs:\n version: ${{ steps.vars.outputs.VERSION }}\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Version\n id: vars\n shell: bash\n run: |\n echo \"VERSION=${{ github.event.inputs.version }}\" >> $GITHUB_OUTPUT\n echo ${{ github.event.inputs.version }} > VERSION\n git add VERSION\n sed -i -e \"s/^\\:project-version\\:\\ .*/:project-version: ${{ github.event.inputs.version }}/g\" README.adoc\n git config --global user.email \"${{ secrets.COMMIT_EMAIL }}\"\n git config --global user.name \"Julien Ruaux\"\n git commit -a -m \"Releasing version ${{ github.event.inputs.version }}\"\n git push origin main\n\n jlink:\n name: Jlink\n needs: [precheck]\n uses: redis/riot/.github/workflows/step-jlink.yml@main\n with:\n project-version: ${{ needs.precheck.outputs.version }}\n tasks: ${{ github.event.inputs.build-tasks }}\n secrets:\n codecov-token: ${{ secrets.CODECOV_TOKEN }}\n\n release:\n name: Release\n needs: [precheck, jlink]\n runs-on: ubuntu-latest\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n with:\n ref: main\n fetch-depth: 0\n\n - name: Download artifacts\n uses: actions/download-artifact@v4\n with:\n name: artifacts\n path: plugins/riot/build\n\n - name: Download repos\n uses: actions/download-artifact@v4\n with:\n name: repos\n path: build/repos/local/release\n\n - name: Download jlink\n uses: actions/download-artifact@v4\n with:\n name: jlink\n path: out/jreleaser/assemble/riot-standalone/jlink\n\n - name: Setup Java\n uses: actions/setup-java@v4\n with:\n java-version: ${{ vars.JAVA_VERSION }}\n distribution: ${{ vars.JAVA_DISTRO }}\n cache: gradle\n\n - name: Deploy\n env:\n GRGIT_USER: ${{ secrets.GIT_USER }}\n GRGIT_PASS: ${{ secrets.GIT_ACCESS_TOKEN }}\n run: |\n ./gradlew -Prelease=true -PreproducibleBuild=true publish gitPublishPush -S\n\n - name: Upload deploy artifacts\n uses: actions/upload-artifact@v4\n with:\n retention-days: 7\n name: deploy\n path: |\n build/repos/local/release/\n\n - name: Release\n uses: jreleaser/release-action@v2\n with:\n arguments: full-release\n env:\n JRELEASER_DOCKER_DEFAULT_USERNAME: ${{ secrets.DOCKER_USERNAME }}\n JRELEASER_DOCKER_DEFAULT_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}\n JRELEASER_GITHUB_PASSWORD: ${{ secrets.GIT_ACCESS_TOKEN }}\n JRELEASER_GITHUB_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}\n JRELEASER_GITHUB_USERNAME: ${{ secrets.GIT_USER }}\n JRELEASER_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}\n JRELEASER_GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }}\n JRELEASER_GPG_SECRET_KEY: ${{ secrets.GPG_SECRET_KEY }}\n JRELEASER_NEXUS2_USERNAME: ${{ secrets.SONATYPE_USERNAME }}\n JRELEASER_NEXUS2_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}\n JRELEASER_PROJECT_VERSION: ${{ needs.precheck.outputs.version }}\n JRELEASER_SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}\n\n - name: JReleaser release output\n if: always()\n uses: actions/upload-artifact@v4\n with:\n name: riot-release\n path: |\n out/jreleaser/trace.log\n out/jreleaser/output.properties" }, { "path": ".github/workflows/step-jlink.yml", "content": "name: X-Jlink\n\non:\n workflow_call:\n inputs:\n project-version:\n required: true\n type: string\n tasks:\n default: 'build aggregateTestReports aggregateJacocoReport publish'\n required: false\n type: string\n secrets:\n codecov-token:\n required: false\n\npermissions:\n contents: read\n\njobs:\n jlink:\n name: Jlink\n runs-on: ubuntu-latest\n env:\n CI: true\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n with:\n ref: main\n\n - name: Setup Java\n uses: actions/setup-java@v4\n with:\n java-version: ${{ vars.JAVA_VERSION }}\n distribution: ${{ vars.JAVA_DISTRO }}\n cache: gradle\n\n - name: Version\n if: endsWith(${{ inputs.project-version }}, '-SNAPSHOT') != true\n run: |\n echo ${{ inputs.project-version }} > VERSION\n\n - name: Build\n run: ./gradlew -Prelease=true -PreproducibleBuild=true ${{ inputs.tasks }} -S\n \n - name: Upload test reports\n if: failure()\n uses: actions/upload-artifact@v4\n with:\n name: test-reports\n path: |\n build/reports/aggregate-tests/\n\n - name: Upload coverage to Codecov\n uses: codecov/codecov-action@v4\n env:\n CODECOV_TOKEN: ${{ secrets.codecov-token }}\n\n - name: Jlink\n uses: jreleaser/release-action@v2\n with:\n arguments: assemble --assembler jlink\n setup-java: false\n env:\n JRELEASER_PROJECT_VERSION: ${{ inputs.project-version }}\n\n - name: JReleaser output\n if: always()\n uses: actions/upload-artifact@v4\n with:\n name: riot-jlink\n path: |\n out/jreleaser/trace.log\n out/jreleaser/output.properties\n\n - name: Dependencies\n run: |\n ls -l plugins/riot/build/dependencies/flat\n \n - name: Upload repos\n uses: actions/upload-artifact@v4\n with:\n retention-days: 1\n name: repos\n path: |\n build/repos/local/release\n\n - name: Upload artifacts\n uses: actions/upload-artifact@v4\n with:\n retention-days: 1\n name: artifacts\n path: |\n plugins/riot/build/libs/\n plugins/riot/build/dependencies/\n plugins/riot/build/distributions/\n\n - name: Stop Gradle daemon\n shell: bash\n run: ./gradlew -stop\n\n - name: Upload jlink\n uses: actions/upload-artifact@v4\n with:\n retention-days: 1\n name: jlink\n path: |\n out/jreleaser/assemble/riot-standalone/jlink/*.zip\n" }, { "path": ".github/workflows/step-precheck.yml", "content": "name: X-Precheck\n\non:\n workflow_call:\n outputs:\n version:\n description: version\n value: ${{ jobs.precheck.outputs.version }}\n\npermissions:\n contents: read\n\njobs:\n precheck:\n name: Precheck\n if: github.repository == 'redis/riot' && startsWith(github.event.head_commit.message, 'Releasing version') != true\n runs-on: ubuntu-latest\n outputs:\n version: ${{ steps.vars.outputs.VERSION }}\n steps:\n - name: Checkout\n uses: actions/checkout@v4\n\n - name: Cancel previous run\n uses: styfle/cancel-workflow-action@0.12.1\n with:\n access_token: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Version\n id: vars\n shell: bash\n run: |\n echo \"VERSION=$(cat VERSION)\" >> $GITHUB_OUTPUT\n" }, { "path": ".github/workflows/trigger-early-access.yml", "content": "name: Trigger Early Access\n\non:\n workflow_dispatch:\n\npermissions:\n contents: read\n\njobs:\n earlyaccess:\n name: Trigger Early Access\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n\n - name: Setup Java\n uses: actions/setup-java@v4\n with:\n java-version: ${{ vars.JAVA_VERSION }}\n distribution: ${{ vars.JAVA_DISTRO }}\n cache: gradle\n\n - name: Build\n run: ./gradlew -PreproducibleBuild=true build -S\n\n - name: Rename artifacts\n run: |\n mkdir early-access\n cp plugins/riot/build/distributions/riot-*-SNAPSHOT.zip early-access/riot-early-access.zip\n\n - name: Release early-access artifacts\n uses: marvinpinto/action-automatic-releases@latest\n with:\n automatic_release_tag: early-access\n repo_token: ${{ secrets.GIT_ACCESS_TOKEN }}\n prerelease: true\n title: RIOT Early-Access\n files: |\n early-access/*\n" }, { "path": ".gitignore", "content": "/out/\nbuild\n.gradle\n/.idea\nbin\n.project\n.classpath\n.settings\n.DS_Store\nhs_err_pid*.log\n*.bkp" }, { "path": "LICENSE", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n" }, { "path": "README.adoc", "content": ":linkattrs:\n:project-owner: redis\n:project-name: riot\n:project-group: com.redis\n:project-version: 4.3.0\n:project-title: RIOT\n\nimage::docs/guide/src/docs/resources/images/riot.svg[RIOT]\n\n---\n\n[IMPORTANT]\n====\n*⚠️ This project is no longer maintained.*\n\nRIOT has been superseded by https://github.com/redis/riotx-dist[*RIOT-X*], an enterprise-grade version with enhanced features and active support from Redis, Inc.\n\n* *For new projects:* Please use https://redis.github.io/riotx/[RIOT-X]\n* *Existing users:* You can continue using RIOT, but no new features or bug fixes will be provided\n* *Support:* All support requests should be directed to the RIOT-X project\n\nhttps://github.com/redis/riotx-dist[→ Go to RIOT-X on GitHub]\n====\n\n---\n\nGet data in and out of Redis with link:http://redis.github.io/riot/[{project-title}]!\n\nRedis Input/Output Tools ({project-title}) is a command-line utility designed to help you get data in and out of Redis.\n\nIt supports many different sources and targets:\n\n* Relational databases\n* Files (CSV, JSON, XML)\n* Data generators (Redis data structures, Faker)\n* and Redis itself with live and/or snapshot replication.\n\n== Documentation\n\nlink:http://redis.github.io/riot/[redis.github.io/riot]\n\n== Support\n\n*This project is no longer actively maintained.*\n\nFor support, please use https://github.com/redis/riotx-dist[RIOT-X], the actively maintained successor to this project.\n\nExisting issues will remain open for historical reference, but new issues and pull requests will not be addressed.\n\n== Badges\n\nimage:https://github.com/{project-owner}/{project-name}/actions/workflows/early-access.yml/badge.svg[\"Build Status\", link=\"https://github.com/{project-owner}/{project-name}/actions/workflows/early-access.yml\"]\nimage:https://img.shields.io/github/release/{project-owner}/{project-name}.svg[\"Latest\", link=\"https://github.com/{project-owner}/{project-name}/releases/latest\"]\nimage:https://codecov.io/gh/{project-owner}/{project-name}/graph/badge.svg?token=bwm6gyXU0v[\"Coverage\", link=\"https://codecov.io/gh/{project-owner}/{project-name}\"]\n\n== License\n\n{project-title} is licensed under the Apache 2.0 License.\nCopyright (C) 2023 Redis, Inc.\n" }, { "path": "VERSION", "content": "4.3.0\n" }, { "path": "build.gradle", "content": "/*\n * SPDX-License-Identifier: Apache-2.0\n *\n * Copyright 2020-2023 The RIOT authors.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * https://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nconfig {\n info {\n description = 'RIOT'\n inceptionYear = '2020'\n vendor = 'Redis'\n tags = ['redis', 'tool', 'import', 'export', 'replication']\n\n links {\n website = \"https://github.com/redis/${project.rootProject.name}\"\n issueTracker = \"https://github.com/redis/${project.rootProject.name}/issues\"\n scm = \"https://github.com/redis/${project.rootProject.name}.git\"\n }\n\n scm {\n url = \"https://github.com/redis/${project.rootProject.name}\"\n connection = \"scm:git:https://github.com/redis/${project.rootProject.name}.git\"\n developerConnection = \"scm:git:git@github.com:redis/${project.rootProject.name}.git\"\n }\n\n specification {\n enabled = true\n }\n\n implementation {\n enabled = true\n }\n \n people {\n clearDomainSet()\n person {\n id = 'jruaux'\n name = 'Julien Ruaux'\n roles = ['developer', 'author']\n }\n }\n }\n\n docs {\n javadoc {\n autoLinks {\n enabled = false\n }\n }\n }\n \n licensing {\n enabled = false\n }\n\n coverage {\n jacoco {\n enabled = true\n toolVersion = jacocoPluginVersion\n }\n }\n}\n\nallprojects {\n repositories {\n mavenLocal()\n maven {\n url 'https://s01.oss.sonatype.org/content/repositories/snapshots/'\n }\n }\n\n tasks.withType(GenerateModuleMetadata) {\n enabled = false\n }\n\n tasks.withType(JavaCompile) {\n options.encoding = 'UTF-8'\n }\n}\n\nsubprojects { subproj ->\n if (!subproj.name.contains('guide')) {\n \n config {\n\t info {\n\t description = project.project_description\n\t }\n coverage {\n jacoco {\n\t\t toolVersion = jacocoPluginVersion\n }\n }\n\t }\n\n\t license {\n\t exclude('build/**')\n\t }\n\n\t dependencies {\n compileOnly 'org.projectlombok:lombok'\n annotationProcessor 'org.projectlombok:lombok'\n testImplementation 'org.junit.jupiter:junit-jupiter-api'\n testImplementation 'org.junit.jupiter:junit-jupiter-params'\n testImplementation 'org.junit.jupiter:junit-jupiter-engine'\n testImplementation 'org.junit.platform:junit-platform-launcher'\n testImplementation 'org.testcontainers:junit-jupiter'\n\t testImplementation group: 'com.redis', name: 'testcontainers-redis', version: testcontainersRedisVersion\n\t }\n\n\t test {\n\t useJUnitPlatform()\n\t }\n\n\t configurations {\n\t all*.exclude module: 'spring-boot-starter-logging'\n\t all*.exclude module: 'commons-logging'\n\t }\n\n\t bootJar {\n enabled = false\n }\n\n jar {\n enabled = true\n archiveClassifier = ''\n }\n\n\t}\n}\n\nsubprojects { p ->\n config {\n coverage {\n jacoco {\n enabled = hasTestsAt(p.file('src/test'))\n }\n }\n }\n \n configurations.all {\n resolutionStrategy.eachDependency { DependencyResolveDetails details ->\n if (details.requested.name == 'lettuce-core' ) {\n details.useVersion lettuceVersion\n }\n }\n }\n}\n\ngradleProjects {\n subprojects {\n dirs(['core']) {\n compileJava {\n options.compilerArgs += [\"-AprojectPath=${project.group}/${project.name}\"]\n }\n\n if (!(project.findProperty('automatic.module.name.skip') ?: false).toBoolean()) {\n jar {\n manifest {\n attributes('Automatic-Module-Name': project.findProperty('automatic.module.name'))\n }\n }\n }\n }\n }\n}\n\nboolean hasTestsAt(File testDir) {\n testDir.exists() && testDir.listFiles()?.size()\n}" }, { "path": "core/riot-core/gradle.properties", "content": "#\n# SPDX-License-Identifier: Apache-2.0\n#\n# Copyright 2022-2023 The RIOT authors.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\nproject_description = RIOT Core\nautomatic.module.name = com.redis.riot.core\n" }, { "path": "core/riot-core/riot-core.gradle", "content": "/*\n * SPDX-License-Identifier: Apache-2.0\n *\n * Copyright 2020-2023 The RIOT authors.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * https://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\ndependencies {\n api 'org.slf4j:slf4j-api'\n implementation 'org.slf4j:slf4j-simple'\n api group: 'info.picocli', name: 'picocli', version: picocliVersion\n annotationProcessor group: 'info.picocli', name: 'picocli-codegen', version: picocliVersion\n implementation 'org.springframework.boot:spring-boot'\n implementation group: 'me.tongfei', name: 'progressbar', version: progressbarVersion\n api group: 'com.redis', name: 'spring-batch-redis-infrastructure', version: springBatchRedisVersion\n testImplementation 'org.awaitility:awaitility'\n}\n\nproject.rootProject.gradle.addBuildListener(new BuildAdapter() {\n @Override\n void projectsEvaluated(Gradle gradle) {\n gradle.rootProject.subprojects\n .find { p -> p.name == 'riot-core' }\n .processResources {\n inputs.property('build_date', gradle.rootProject.config.buildInfo.buildDate + ':' + gradle.rootProject.config.buildInfo.buildTime)\n filesMatching(['**/RiotVersion.properties']) {\n expand(\n 'riot_version': gradle.rootProject.version,\n 'build_date': gradle.rootProject.config.buildInfo.buildDate,\n 'build_time': gradle.rootProject.config.buildInfo.buildTime,\n 'build_revision': gradle.rootProject.config.buildInfo.buildRevision\n )\n }\n }\n }\n})" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/AbstractCallableCommand.java", "content": "package com.redis.riot.core;\n\nimport java.util.concurrent.Callable;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport picocli.CommandLine.Command;\nimport picocli.CommandLine.Option;\n\n@Command\npublic abstract class AbstractCallableCommand extends BaseCommand implements Callable {\n\n\t@Option(names = \"--help\", usageHelp = true, description = \"Show this help message and exit.\")\n\tprivate boolean helpRequested;\n\n\tprotected Logger log;\n\n\t@Override\n\tpublic Integer call() throws Exception {\n\t\tinitialize();\n\t\ttry {\n\t\t\texecute();\n\t\t} finally {\n\t\t\tteardown();\n\t\t}\n\t\treturn 0;\n\t}\n\n\tprotected void initialize() {\n\t\tif (log == null) {\n\t\t\tlog = LoggerFactory.getLogger(getClass());\n\t\t}\n\t}\n\n\tprotected abstract void execute();\n\n\tprotected void teardown() {\n\t\t// do nothing\n\t}\n\n\tpublic Logger getLog() {\n\t\treturn log;\n\t}\n\n\tpublic void setLog(Logger log) {\n\t\tthis.log = log;\n\t}\n\n}\n" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/AbstractJobCommand.java", "content": "package com.redis.riot.core;\n\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.Iterator;\nimport java.util.List;\n\nimport org.springframework.batch.core.BatchStatus;\nimport org.springframework.batch.core.ExitStatus;\nimport org.springframework.batch.core.ItemWriteListener;\nimport org.springframework.batch.core.Job;\nimport org.springframework.batch.core.JobExecution;\nimport org.springframework.batch.core.JobExecutionException;\nimport org.springframework.batch.core.JobExecutionListener;\nimport org.springframework.batch.core.JobParameters;\nimport org.springframework.batch.core.JobParametersBuilder;\nimport org.springframework.batch.core.JobParametersInvalidException;\nimport org.springframework.batch.core.StepExecution;\nimport org.springframework.batch.core.StepExecutionListener;\nimport org.springframework.batch.core.explore.JobExplorer;\nimport org.springframework.batch.core.job.builder.JobBuilder;\nimport org.springframework.batch.core.job.builder.SimpleJobBuilder;\nimport org.springframework.batch.core.launch.JobLauncher;\nimport org.springframework.batch.core.launch.support.RunIdIncrementer;\nimport org.springframework.batch.core.launch.support.TaskExecutorJobLauncher;\nimport org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;\nimport org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;\nimport org.springframework.batch.core.repository.JobRepository;\nimport org.springframework.batch.core.repository.JobRestartException;\nimport org.springframework.batch.core.step.builder.FaultTolerantStepBuilder;\nimport org.springframework.batch.core.step.builder.SimpleStepBuilder;\nimport org.springframework.batch.core.step.builder.StepBuilder;\nimport org.springframework.batch.core.step.skip.AlwaysSkipItemSkipPolicy;\nimport org.springframework.batch.core.step.skip.NeverSkipItemSkipPolicy;\nimport org.springframework.batch.core.step.tasklet.TaskletStep;\nimport org.springframework.batch.item.ItemReader;\nimport org.springframework.batch.item.ItemStreamReader;\nimport org.springframework.batch.item.ItemStreamSupport;\nimport org.springframework.batch.item.ItemWriter;\nimport org.springframework.batch.item.support.SynchronizedItemReader;\nimport org.springframework.batch.item.support.SynchronizedItemStreamReader;\nimport org.springframework.core.task.SyncTaskExecutor;\nimport org.springframework.core.task.TaskExecutor;\nimport org.springframework.retry.policy.AlwaysRetryPolicy;\nimport org.springframework.retry.policy.NeverRetryPolicy;\nimport org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;\nimport org.springframework.transaction.PlatformTransactionManager;\nimport org.springframework.util.Assert;\nimport org.springframework.util.ClassUtils;\nimport org.springframework.util.CollectionUtils;\n\nimport com.redis.spring.batch.JobUtils;\nimport com.redis.spring.batch.item.AbstractAsyncItemStreamSupport;\nimport com.redis.spring.batch.item.PollableItemReader;\nimport com.redis.spring.batch.step.FlushingStepBuilder;\n\nimport picocli.CommandLine.ArgGroup;\nimport picocli.CommandLine.Command;\nimport picocli.CommandLine.Option;\n\n@Command\npublic abstract class AbstractJobCommand extends AbstractCallableCommand {\n\n\tpublic static final String DEFAULT_JOB_REPOSITORY_NAME = \"riot\";\n\n\t@Option(names = \"--job-name\", description = \"Job name.\", paramLabel = \"\", hidden = true)\n\tprivate String jobName;\n\n\t@Option(names = \"--repeat\", description = \"After the job completes keep repeating it on a fixed interval (ex 5m, 1h)\", paramLabel = \"\")\n\tprivate RiotDuration repeatEvery;\n\n\t@ArgGroup(exclusive = false, heading = \"Job options%n\")\n\tprivate StepArgs stepArgs = new StepArgs();\n\n\tprivate String jobRepositoryName = DEFAULT_JOB_REPOSITORY_NAME;\n\tprivate JobRepository jobRepository;\n\tprivate PlatformTransactionManager transactionManager;\n\tprivate JobLauncher jobLauncher;\n\tprivate JobExplorer jobExplorer;\n\n\tprotected Runnable onJobSuccessCallback;\n\n\t@Override\n\tprotected void initialize() {\n\t\tsuper.initialize();\n\t\tif (jobName == null) {\n\t\t\tjobName = jobName();\n\t\t}\n\t\tif (jobRepository == null) {\n\t\t\ttry {\n\t\t\t\tjobRepository = JobUtils.jobRepositoryFactoryBean(jobRepositoryName).getObject();\n\t\t\t} catch (Exception e) {\n\t\t\t\tthrow new RiotException(\"Could not create job repository\", e);\n\t\t\t}\n\t\t}\n\t\tif (transactionManager == null) {\n\t\t\ttransactionManager = JobUtils.resourcelessTransactionManager();\n\t\t}\n\t\tif (jobLauncher == null) {\n\t\t\ttry {\n\t\t\t\tjobLauncher = jobLauncher();\n\t\t\t} catch (Exception e) {\n\t\t\t\tthrow new RiotException(\"Could not create job launcher\", e);\n\t\t\t}\n\t\t}\n\t\tif (jobExplorer == null) {\n\t\t\ttry {\n\t\t\t\tjobExplorer = JobUtils.jobExplorerFactoryBean(jobRepositoryName).getObject();\n\t\t\t} catch (Exception e) {\n\t\t\t\tlog.warn(\"Error getting jobExplorer\", e);\n\t\t\t\tthrow new RiotException(\"Could not create job explorer\", e);\n\t\t\t}\n\t\t}\n\t}\n\n\tprivate JobLauncher jobLauncher() throws Exception {\n\t\tTaskExecutorJobLauncher launcher = new TaskExecutorJobLauncher();\n\t\tlauncher.setJobRepository(jobRepository);\n\t\tlauncher.setTaskExecutor(new SyncTaskExecutor());\n\t\tlauncher.afterPropertiesSet();\n\t\treturn launcher;\n\t}\n\n\tprotected void configureAsyncStreamSupport(AbstractAsyncItemStreamSupport reader) {\n\t\treader.setJobRepository(jobRepository);\n\t}\n\n\tprivate JobBuilder jobBuilder() {\n\t\treturn new JobBuilder(jobName, jobRepository);\n\t}\n\n\t@Override\n\tprotected void execute() {\n\t\tJob job = job();\n\t\tJobExecution jobExecution;\n\t\ttry {\n\t\t\tjobExecution = jobLauncher.run(job, new JobParameters());\n\t\t} catch (JobExecutionException e) {\n\t\t\tthrow new RiotException(e);\n\t\t}\n\t\tif (JobUtils.isFailed(jobExecution.getExitStatus())) {\n\t\t\tfor (StepExecution stepExecution : jobExecution.getStepExecutions()) {\n\t\t\t\tExitStatus stepExitStatus = stepExecution.getExitStatus();\n\t\t\t\tif (JobUtils.isFailed(stepExitStatus)) {\n\t\t\t\t\tif (CollectionUtils.isEmpty(stepExecution.getFailureExceptions())) {\n\t\t\t\t\t\tthrow new RiotException(stepExitStatus.getExitDescription());\n\t\t\t\t\t}\n\t\t\t\t\tthrow wrapException(stepExecution.getFailureExceptions());\n\t\t\t\t}\n\t\t\t}\n\t\t\tthrow wrapException(jobExecution.getFailureExceptions());\n\t\t}\n\t}\n\n\tprivate String jobName() {\n\t\tif (commandSpec == null) {\n\t\t\treturn ClassUtils.getShortName(getClass());\n\t\t}\n\t\treturn commandSpec.name();\n\t}\n\n\tprivate RiotException wrapException(List throwables) {\n\t\tif (throwables.isEmpty()) {\n\t\t\treturn new RiotException(\"Job failed\");\n\t\t}\n\t\treturn new RiotException(throwables.get(0));\n\t}\n\n\tprotected Job job(Step... steps) {\n\t\treturn job(Arrays.asList(steps));\n\t}\n\n\tprotected Job job(Collection> steps) {\n\t\tAssert.notEmpty(steps, \"At least one step must be specified\");\n\t\tIterator> iterator = steps.iterator();\n\t\tSimpleJobBuilder job = jobBuilder().start(step(iterator.next()));\n\t\twhile (iterator.hasNext()) {\n\t\t\tjob.next(step(iterator.next()));\n\t\t}\n\n\t\tif (repeatEvery != null) {\n\t\t\tjob.incrementer(new RunIdIncrementer());\n\t\t\tjob.preventRestart();\n\t\t\tjob.listener(new RepeatJobExecutionListener(job, steps));\n\t\t}\n\n\t\treturn job.build();\n\t}\n\n\tprivate class RepeatJobExecutionListener implements JobExecutionListener {\n\n\t\tprivate final SimpleJobBuilder job;\n\t\tprivate final Collection> steps;\n\t\tprivate Job lastJob;\n\n\t\tpublic RepeatJobExecutionListener(SimpleJobBuilder job, Collection> steps) {\n\t\t\tthis.job = job;\n\t\t\tthis.steps = steps;\n\t\t}\n\n\t\t@Override\n\t\tpublic void afterJob(JobExecution jobExecution) {\n\t\t\tif (jobExecution.getStatus() == BatchStatus.COMPLETED) {\n\t\t\t\tif (null != onJobSuccessCallback) {\n\t\t\t\t\tonJobSuccessCallback.run();\n\t\t\t\t}\n\n\t\t\t\tlog.info(\"Finished job, will run again in {}\", repeatEvery);\n\t\t\t\ttry {\n\t\t\t\t\tThread.sleep(repeatEvery.getValue().toMillis());\n\t\t\t\t\tif (lastJob == null) {\n\t\t\t\t\t\tlastJob = job.build();\n\t\t\t\t\t}\n\n\t\t\t\t\tJob nextJob = jobBuilder().start(step(steps.stream().findFirst().get()))\n\t\t\t\t\t\t\t.incrementer(new RunIdIncrementer()).preventRestart().listener(this).build();\n\n\t\t\t\t\tJobParametersBuilder paramsBuilder = new JobParametersBuilder(jobExecution.getJobParameters(),\n\t\t\t\t\t\t\tjobExplorer);\n\n\t\t\t\t\tjobLauncher.run(nextJob,\n\t\t\t\t\t\t\tparamsBuilder.addString(\"runTime\", String.valueOf(System.currentTimeMillis()))\n\t\t\t\t\t\t\t\t\t.getNextJobParameters(lastJob).toJobParameters());\n\t\t\t\t\tlastJob = nextJob;\n\t\t\t\t} catch (InterruptedException | JobExecutionAlreadyRunningException | JobRestartException\n\t\t\t\t\t\t| JobInstanceAlreadyCompleteException | JobParametersInvalidException e) {\n\t\t\t\t\tthrow new RiotException(e);\n\t\t\t\t}\n\t\t\t}\n\t\t\tJobExecutionListener.super.afterJob(jobExecution);\n\t\t}\n\t}\n\n\tprotected boolean shouldShowProgress() {\n\t\treturn stepArgs.getProgressArgs().getStyle() != ProgressStyle.NONE;\n\t}\n\n\tprotected abstract Job job();\n\n\tprivate TaskletStep step(Step step) {\n\t\tlog.info(\"Creating {}\", step);\n\t\tSimpleStepBuilder builder = simpleStep(step);\n\t\tif (stepArgs.getRetryPolicy() == RetryPolicy.NEVER && stepArgs.getSkipPolicy() == SkipPolicy.NEVER) {\n\t\t\tlog.info(\"Skipping fault-tolerance for step {}\", step.getName());\n\t\t\treturn builder.build();\n\t\t}\n\t\tlog.info(\"Adding fault-tolerance to step {}\", step.getName());\n\t\tFaultTolerantStepBuilder ftStep = builder.faultTolerant();\n\t\tstep.getSkip().forEach(ftStep::skip);\n\t\tstep.getNoSkip().forEach(ftStep::noSkip);\n\t\tstep.getRetry().forEach(ftStep::retry);\n\t\tstep.getNoRetry().forEach(ftStep::noRetry);\n\t\tftStep.retryLimit(stepArgs.getRetryLimit());\n\t\tftStep.retryPolicy(retryPolicy());\n\t\tftStep.skipLimit(stepArgs.getSkipLimit());\n\t\tftStep.skipPolicy(skipPolicy());\n\t\treturn ftStep.build();\n\t}\n\n\tprivate org.springframework.retry.RetryPolicy retryPolicy() {\n\t\tswitch (stepArgs.getRetryPolicy()) {\n\t\tcase ALWAYS:\n\t\t\treturn new AlwaysRetryPolicy();\n\t\tcase NEVER:\n\t\t\treturn new NeverRetryPolicy();\n\t\tdefault:\n\t\t\treturn null;\n\t\t}\n\t}\n\n\tprivate org.springframework.batch.core.step.skip.SkipPolicy skipPolicy() {\n\t\tswitch (stepArgs.getSkipPolicy()) {\n\t\tcase ALWAYS:\n\t\t\treturn new AlwaysSkipItemSkipPolicy();\n\t\tcase NEVER:\n\t\t\treturn new NeverSkipItemSkipPolicy();\n\t\tdefault:\n\t\t\treturn null;\n\t\t}\n\t}\n\n\t@SuppressWarnings(\"removal\")\n\tprivate SimpleStepBuilder simpleStep(Step step) {\n\t\tString stepName = jobName + \"-\" + step.getName();\n\t\tif (stepName.length() > 80) {\n\t\t\tstepName = stepName.substring(0, 69) + \"…\" + stepName.substring(stepName.length() - 10);\n\t\t}\n\t\tif (step.getReader() instanceof ItemStreamSupport) {\n\t\t\t((ItemStreamSupport) step.getReader()).setName(stepName + \"-reader\");\n\t\t}\n\t\tlog.info(\"Creating step {} with chunk size {}\", stepName, stepArgs.getChunkSize());\n\t\tSimpleStepBuilder builder = new StepBuilder(stepName, jobRepository).chunk(stepArgs.getChunkSize(),\n\t\t\t\ttransactionManager);\n\t\tbuilder.reader(reader(step));\n\t\tbuilder.writer(writer(step));\n\t\tbuilder.processor(step.getProcessor());\n\t\tbuilder.taskExecutor(taskExecutor());\n\t\tbuilder.throttleLimit(stepArgs.getThreads());\n\t\tstep.getExecutionListeners().forEach(builder::listener);\n\t\tstep.getWriteListeners().forEach(builder::listener);\n\t\tif (shouldShowProgress()) {\n\t\t\tProgressStepExecutionListener listener = new ProgressStepExecutionListener<>(step);\n\t\t\tbuilder.listener((StepExecutionListener) listener);\n\t\t\tbuilder.listener((ItemWriteListener) listener);\n\t\t}\n\t\tif (step.isLive()) {\n\t\t\tlog.info(\"Creating flushing step with flush interval {} and idle timeout {}\", step.getFlushInterval(),\n\t\t\t\t\tstep.getIdleTimeout());\n\t\t\tFlushingStepBuilder flushingStepBuilder = new FlushingStepBuilder<>(builder);\n\t\t\tflushingStepBuilder.flushInterval(step.getFlushInterval());\n\t\t\tflushingStepBuilder.idleTimeout(step.getIdleTimeout());\n\t\t\treturn flushingStepBuilder;\n\t\t}\n\t\treturn builder;\n\t}\n\n\tprivate TaskExecutor taskExecutor() {\n\t\tif (stepArgs.getThreads() == 1) {\n\t\t\treturn new SyncTaskExecutor();\n\t\t}\n\t\tlog.info(\"Creating thread-pool task executor of size {}\", stepArgs.getThreads());\n\t\tThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();\n\t\ttaskExecutor.setMaxPoolSize(stepArgs.getThreads());\n\t\ttaskExecutor.setCorePoolSize(stepArgs.getThreads());\n\t\ttaskExecutor.initialize();\n\t\treturn taskExecutor;\n\t}\n\n\tprivate ItemReader reader(Step step) {\n\t\tif (stepArgs.getThreads() == 1 || step.getReader() instanceof PollableItemReader) {\n\t\t\treturn step.getReader();\n\t\t}\n\t\tlog.info(\"Synchronizing reader in step {}\", step.getName());\n\t\tif (step.getReader() instanceof ItemStreamReader) {\n\t\t\tSynchronizedItemStreamReader synchronizedReader = new SynchronizedItemStreamReader<>();\n\t\t\tsynchronizedReader.setDelegate((ItemStreamReader) step.getReader());\n\t\t\treturn synchronizedReader;\n\t\t}\n\t\treturn new SynchronizedItemReader<>(step.getReader());\n\t}\n\n\tprivate ItemWriter writer(Step step) {\n\t\tItemWriter writer = step.getWriter();\n\t\tif (stepArgs.isDryRun()) {\n\t\t\tlog.info(\"Using no-op writer\");\n\t\t\twriter = new NoopItemWriter<>();\n\t\t}\n\t\tif (stepArgs.getSleep() != null) {\n\t\t\tlog.info(\"Throttling writer with sleep={}\", stepArgs.getSleep());\n\t\t\twriter = new ThrottledItemWriter<>(writer, stepArgs.getSleep().getValue());\n\t\t}\n\t\treturn writer;\n\t}\n\n\tpublic String getJobName() {\n\t\treturn jobName;\n\t}\n\n\tpublic void setJobName(String name) {\n\t\tthis.jobName = name;\n\t}\n\n\tpublic StepArgs getJobArgs() {\n\t\treturn stepArgs;\n\t}\n\n\tpublic void setJobArgs(StepArgs args) {\n\t\tthis.stepArgs = args;\n\t}\n\n\tpublic String getJobRepositoryName() {\n\t\treturn jobRepositoryName;\n\t}\n\n\tpublic void setJobRepositoryName(String jobRepositoryName) {\n\t\tthis.jobRepositoryName = jobRepositoryName;\n\t}\n\n\tpublic JobRepository getJobRepository() {\n\t\treturn jobRepository;\n\t}\n\n\tpublic void setJobRepository(JobRepository jobRepository) {\n\t\tthis.jobRepository = jobRepository;\n\t}\n\n\tpublic PlatformTransactionManager getTransactionManager() {\n\t\treturn transactionManager;\n\t}\n\n\tpublic void setTransactionManager(PlatformTransactionManager transactionManager) {\n\t\tthis.transactionManager = transactionManager;\n\t}\n\n\tpublic JobLauncher getJobLauncher() {\n\t\treturn jobLauncher;\n\t}\n\n\tpublic void setJobLauncher(JobLauncher jobLauncher) {\n\t\tthis.jobLauncher = jobLauncher;\n\t}\n\n\tpublic JobExplorer getJobExplorer() {\n\t\treturn jobExplorer;\n\t}\n\n\tpublic void setJobExplorer(JobExplorer jobExplorer) {\n\t\tthis.jobExplorer = jobExplorer;\n\t}\n}\n" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/BaseCommand.java", "content": "package com.redis.riot.core;\n\nimport java.util.Map;\n\nimport picocli.CommandLine.Command;\nimport picocli.CommandLine.Mixin;\nimport picocli.CommandLine.Model.CommandSpec;\nimport picocli.CommandLine.Option;\nimport picocli.CommandLine.Spec;\n\n@Command(usageHelpAutoWidth = true, mixinStandardHelpOptions = true, abbreviateSynopsis = true)\npublic class BaseCommand {\n\n\tstatic {\n\t\tif (System.getenv().containsKey(\"RIOT_NO_COLOR\")) {\n\t\t\tSystem.setProperty(\"picocli.ansi\", \"false\");\n\t\t}\n\t}\n\n\t@Spec\n\tprotected CommandSpec commandSpec;\n\n\t@Mixin\n\tLoggingMixin loggingMixin;\n\n\t@Option(names = \"-D\", paramLabel = \"\", description = \"Sets a System property.\", mapFallbackValue = \"\", hidden = true)\n\tvoid setProperty(Map props) {\n\t\tprops.forEach(System::setProperty);\n\t}\n\n}\n" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/CompositeExecutionStrategy.java", "content": "package com.redis.riot.core;\n\nimport picocli.CommandLine.ExecutionException;\nimport picocli.CommandLine.IExecutionStrategy;\nimport picocli.CommandLine.ParameterException;\nimport picocli.CommandLine.ParseResult;\n\npublic class CompositeExecutionStrategy implements IExecutionStrategy {\n\n\tprivate final IExecutionStrategy inactive;\n\tprivate final IExecutionStrategy active;\n\n\tpublic CompositeExecutionStrategy(IExecutionStrategy inactive, IExecutionStrategy active) {\n\t\tthis.inactive = inactive;\n\t\tthis.active = active;\n\t}\n\n\t@Override\n\tpublic int execute(ParseResult parseResult) throws ExecutionException, ParameterException {\n\t\tinactive.execute(parseResult);\n\t\treturn active.execute(parseResult);\n\t}\n\n}\n" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/Expression.java", "content": "package com.redis.riot.core;\n\nimport java.util.function.Predicate;\n\nimport org.springframework.expression.EvaluationContext;\nimport org.springframework.expression.common.TemplateParserContext;\nimport org.springframework.expression.spel.standard.SpelExpressionParser;\n\npublic class Expression {\n\n\tprotected static final SpelExpressionParser PARSER = new SpelExpressionParser();\n\n\tprotected final org.springframework.expression.Expression spelExpression;\n\n\tpublic Expression(org.springframework.expression.Expression expression) {\n\t\tthis.spelExpression = expression;\n\t}\n\n\t@Override\n\tpublic String toString() {\n\t\treturn spelExpression.getExpressionString();\n\t}\n\n\tpublic Predicate predicate(EvaluationContext context) {\n\t\treturn t -> spelExpression.getValue(context, t, Boolean.class);\n\t}\n\n\tpublic Object getValue(EvaluationContext context) {\n\t\treturn spelExpression.getValue(context);\n\t}\n\n\tpublic Object getValue(EvaluationContext context, Object rootObject) {\n\t\treturn spelExpression.getValue(context, rootObject);\n\t}\n\n\tpublic Long getLong(EvaluationContext context, Object rootObject) {\n\t\treturn spelExpression.getValue(context, rootObject, Long.class);\n\t}\n\n\tpublic String getString(EvaluationContext context, Object rootObject) {\n\t\treturn spelExpression.getValue(context, rootObject, String.class);\n\t}\n\n\tpublic static Expression parse(String expression) {\n\t\treturn new Expression(PARSER.parseExpression(expression));\n\t}\n\n\tpublic static TemplateExpression parseTemplate(String expression) {\n\t\treturn new TemplateExpression(PARSER.parseExpression(expression, new TemplateParserContext()));\n\t}\n\n}\n" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/IO.java", "content": "package com.redis.riot.core;\n\nimport java.io.PrintWriter;\n\npublic interface IO {\n\n\tPrintWriter getOut();\n\n\tvoid setOut(PrintWriter out);\n\n\tPrintWriter getErr();\n\n\tvoid setErr(PrintWriter err);\n\n}\n" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/LoggingMixin.java", "content": "package com.redis.riot.core;\n\nimport static picocli.CommandLine.Spec.Target.MIXEE;\n\nimport java.util.LinkedHashMap;\nimport java.util.Map;\n\nimport org.slf4j.event.Level;\nimport org.slf4j.simple.SimpleLogger;\n\nimport picocli.CommandLine.Model.CommandSpec;\nimport picocli.CommandLine.Option;\nimport picocli.CommandLine.ParseResult;\nimport picocli.CommandLine.Spec;\n\npublic class LoggingMixin {\n\n\tpublic static final boolean DEFAULT_SHOW_DATE_TIME = true;\n\tpublic static final boolean DEFAULT_SHOW_THREAD_NAME = true;\n\tpublic static final boolean DEFAULT_SHOW_LOG_NAME = true;\n\tpublic static final Level DEFAULT_LEVEL = Level.WARN;\n\n\t/**\n\t * This mixin is able to climb the command hierarchy because the\n\t * {@code @Spec(Target.MIXEE)}-annotated field gets a reference to the command\n\t * where it is used.\n\t */\n\tprivate @Spec(MIXEE) CommandSpec mixee; // spec of the command where the @Mixin is used\n\n\tprivate String file;\n\tprivate boolean showDateTime = DEFAULT_SHOW_DATE_TIME;\n\tprivate String dateTimeFormat;\n\tprivate boolean showThreadId;\n\tprivate boolean showThreadName = DEFAULT_SHOW_THREAD_NAME;\n\tprivate boolean showLogName = DEFAULT_SHOW_LOG_NAME;\n\tprivate boolean showShortLogName;\n\tprivate boolean levelInBrackets;\n\tprivate Map levels = new LinkedHashMap<>();\n\tprivate Level level = DEFAULT_LEVEL;\n\n\tprivate static LoggingMixin getTopLevelCommandLoggingMixin(CommandSpec commandSpec) {\n\t\treturn ((MainCommand) commandSpec.root().userObject()).loggingMixin;\n\t}\n\n\t@Option(names = \"--log-file\", description = \"Log output target. Can be a path or special values System.out and System.err (default: System.err).\", paramLabel = \"\")\n\tpublic void setFile(String file) {\n\t\tgetTopLevelCommandLoggingMixin(mixee).file = file;\n\t}\n\n\t@Option(names = \"--log-time\", description = \"Include current date and time in log messages. True by default.\", negatable = true, defaultValue = \"true\", fallbackValue = \"true\")\n\tpublic void setShowDateTime(boolean show) {\n\t\tgetTopLevelCommandLoggingMixin(mixee).showDateTime = show;\n\t}\n\n\t@Option(names = \"--log-time-fmt\", defaultValue = \"yyyy-MM-dd HH:mm:ss.SSS\", description = \"Date and time format to be used in log messages (default: ${DEFAULT-VALUE}). Use with --log-time.\", paramLabel = \"\")\n\tpublic void setDateTimeFormat(String format) {\n\t\tgetTopLevelCommandLoggingMixin(mixee).dateTimeFormat = format;\n\t}\n\n\t@Option(names = \"--log-thread-id\", description = \"Include current thread ID in log messages.\", hidden = true)\n\tpublic void setShowThreadId(boolean show) {\n\t\tgetTopLevelCommandLoggingMixin(mixee).showThreadId = show;\n\t}\n\n\t@Option(names = \"--log-thread\", description = \"Show current thread name in log messages. True by default.\", negatable = true, defaultValue = \"true\", fallbackValue = \"true\", hidden = true)\n\tpublic void setShowThreadName(boolean show) {\n\t\tgetTopLevelCommandLoggingMixin(mixee).showThreadName = show;\n\t}\n\n\t@Option(names = \"--log-name\", description = \"Show logger instance name in log messages. True by default.\", negatable = true, defaultValue = \"true\", fallbackValue = \"true\", hidden = true)\n\tpublic void setShowLogName(boolean show) {\n\t\tgetTopLevelCommandLoggingMixin(mixee).showLogName = show;\n\t}\n\n\t@Option(names = \"--log-short\", description = \"Include last component of logger instance name in log messages. True by default.\", negatable = true, defaultValue = \"true\", fallbackValue = \"true\", hidden = true)\n\tpublic void setShowShortLogName(boolean show) {\n\t\tgetTopLevelCommandLoggingMixin(mixee).showShortLogName = show;\n\t}\n\n\t@Option(names = \"--log-level-brck\", description = \"Output log level string in brackets.\", hidden = true)\n\tpublic void setLevelInBrackets(boolean enable) {\n\t\tgetTopLevelCommandLoggingMixin(mixee).levelInBrackets = enable;\n\t}\n\n\t@Option(arity = \"1..*\", names = \"--log\", description = \"Custom log levels (e.g.: io.lettuce=INFO).\", paramLabel = \"\")\n\tpublic void setLevels(Map levels) {\n\t\tgetTopLevelCommandLoggingMixin(mixee).levels = levels;\n\t}\n\n\t@Option(names = { \"-d\", \"--debug\" }, description = \"Log in debug mode.\")\n\tpublic void setDebug(boolean enable) {\n\t\tif (enable) {\n\t\t\tgetTopLevelCommandLoggingMixin(mixee).level = Level.DEBUG;\n\t\t}\n\t}\n\n\t@Option(names = { \"-i\", \"--info\" }, description = \"Set log level to info.\")\n\tpublic void setInfo(boolean enable) {\n\t\tif (enable) {\n\t\t\tgetTopLevelCommandLoggingMixin(mixee).level = Level.INFO;\n\t\t}\n\t}\n\n\t@Option(names = { \"-q\", \"--quiet\" }, description = \"Log errors only.\")\n\tpublic void setError(boolean enable) {\n\t\tif (enable) {\n\t\t\tgetTopLevelCommandLoggingMixin(mixee).level = Level.ERROR;\n\t\t}\n\t}\n\n\t@Option(names = \"--log-level\", description = \"Set log level: ${COMPLETION-CANDIDATES} (default: WARN).\", paramLabel = \"\", hidden = true)\n\tpublic void setLevel(Level level) {\n\t\tgetTopLevelCommandLoggingMixin(mixee).level = level;\n\t}\n\n\tpublic static int executionStrategy(ParseResult parseResult) {\n\t\tgetTopLevelCommandLoggingMixin(parseResult.commandSpec()).configureLoggers();\n\t\treturn 0;\n\t}\n\n\tpublic void configureLoggers() {\n\t\tLoggingMixin mixin = getTopLevelCommandLoggingMixin(mixee);\n\t\tSystem.setProperty(SimpleLogger.DEFAULT_LOG_LEVEL_KEY, mixin.level.name());\n\t\tif (mixin.file != null) {\n\t\t\tSystem.setProperty(SimpleLogger.LOG_FILE_KEY, mixin.file);\n\t\t}\n\t\tsetBoolean(SimpleLogger.SHOW_DATE_TIME_KEY, mixin.showDateTime);\n\t\tif (mixin.dateTimeFormat != null) {\n\t\t\tSystem.setProperty(SimpleLogger.DATE_TIME_FORMAT_KEY, mixin.dateTimeFormat);\n\t\t}\n\t\tsetBoolean(SimpleLogger.SHOW_THREAD_ID_KEY, mixin.showThreadId);\n\t\tsetBoolean(SimpleLogger.SHOW_THREAD_NAME_KEY, mixin.showThreadName);\n\t\tsetBoolean(SimpleLogger.SHOW_LOG_NAME_KEY, mixin.showLogName);\n\t\tsetBoolean(SimpleLogger.SHOW_SHORT_LOG_NAME_KEY, mixin.showShortLogName);\n\t\tsetBoolean(SimpleLogger.LEVEL_IN_BRACKETS_KEY, mixin.levelInBrackets);\n\t\tsetLogLevel(\"com.amazonaws.internal\", Level.ERROR);\n\t\tsetLogLevel(\"com.redis.spring.batch.step.FlushingFaultTolerantStepBuilder\", Level.ERROR);\n\t\tsetLogLevel(\"org.springframework.batch.core.step.builder.FaultTolerantStepBuilder\", Level.ERROR);\n\t\tsetLogLevel(\"org.springframework.batch.core.step.item.ChunkMonitor\", Level.ERROR);\n\t\tmixin.levels.forEach(this::setLogLevel);\n\t}\n\n\tprivate void setLogLevel(String key, Level level) {\n\t\tSystem.setProperty(SimpleLogger.LOG_KEY_PREFIX + key, level.name());\n\t}\n\n\tprivate void setBoolean(String property, boolean value) {\n\t\tSystem.setProperty(property, String.valueOf(value));\n\t}\n\n\tpublic boolean isStacktrace() {\n\t\treturn level.toInt() <= Level.INFO.toInt();\n\t}\n\n}" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/MainCommand.java", "content": "package com.redis.riot.core;\n\nimport java.io.PrintWriter;\nimport java.util.concurrent.Callable;\n\nimport org.springframework.util.unit.DataSize;\n\nimport picocli.CommandLine;\nimport picocli.CommandLine.ParseResult;\nimport picocli.CommandLine.RunLast;\n\npublic class MainCommand extends BaseCommand implements Callable, IO {\n\n\tprivate PrintWriter out;\n\tprivate PrintWriter err;\n\n\t@Override\n\tpublic Integer call() throws Exception {\n\t\tcommandSpec.commandLine().usage(out);\n\t\treturn 0;\n\t}\n\n\tprotected CommandLine commandLine() {\n\t\treturn new CommandLine(this);\n\t}\n\n\tpublic int run(String... args) {\n\t\tCommandLine commandLine = commandLine();\n\t\tsetOut(commandLine.getOut());\n\t\tsetErr(commandLine.getErr());\n\t\tcommandLine.setCaseInsensitiveEnumValuesAllowed(true);\n\t\tcommandLine.setUnmatchedOptionsAllowedAsOptionParameters(false);\n\t\tcommandLine.setExecutionExceptionHandler(new PrintExceptionMessageHandler());\n\t\tregisterConverters(commandLine);\n\t\tcommandLine.setExecutionStrategy(\n\t\t\t\tnew CompositeExecutionStrategy(LoggingMixin::executionStrategy, this::executionStrategy));\n\t\treturn commandLine.execute(args);\n\t}\n\n\tprotected int executionStrategy(ParseResult parseResult) {\n\t\treturn new RunLast().execute(parseResult);\n\t}\n\n\tprotected void registerConverters(CommandLine commandLine) {\n\t\tcommandLine.registerConverter(RiotDuration.class, RiotDuration::parse);\n\t\tcommandLine.registerConverter(DataSize.class, MainCommand::parseDataSize);\n\t\tcommandLine.registerConverter(Expression.class, Expression::parse);\n\t\tcommandLine.registerConverter(TemplateExpression.class, Expression::parseTemplate);\n\t}\n\n\tpublic static DataSize parseDataSize(String string) {\n\t\treturn DataSize.parse(string.toUpperCase());\n\t}\n\n\t@Override\n\tpublic PrintWriter getOut() {\n\t\treturn out;\n\t}\n\n\t@Override\n\tpublic void setOut(PrintWriter out) {\n\t\tthis.out = out;\n\t}\n\n\t@Override\n\tpublic PrintWriter getErr() {\n\t\treturn err;\n\t}\n\n\t@Override\n\tpublic void setErr(PrintWriter err) {\n\t\tthis.err = err;\n\t}\n\n}\n" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/NoopItemWriter.java", "content": "package com.redis.riot.core;\n\nimport org.springframework.batch.item.Chunk;\nimport org.springframework.batch.item.ItemWriter;\n\npublic class NoopItemWriter implements ItemWriter {\n\n\t@Override\n\tpublic void write(Chunk items) {\n\t\t// Do nothing\n\t}\n\n}\n" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/PrintExceptionMessageHandler.java", "content": "package com.redis.riot.core;\n\nimport picocli.CommandLine;\nimport picocli.CommandLine.IExecutionExceptionHandler;\nimport picocli.CommandLine.ParseResult;\n\npublic class PrintExceptionMessageHandler implements IExecutionExceptionHandler {\n\n\tpublic int handleExecutionException(Exception exception, CommandLine cmd, ParseResult parseResult) {\n\n\t\tThrowable finalException = unwrapException(exception);\n\n\t\tif (cmd.getCommand() instanceof BaseCommand) {\n\t\t\tif (((BaseCommand) cmd.getCommand()).loggingMixin.isStacktrace()) {\n\t\t\t\tfinalException.printStackTrace(cmd.getErr());\n\t\t\t}\n\t\t}\n\n\t\t// bold red error message\n\t\tcmd.getErr().println(cmd.getColorScheme().errorText(finalException.getMessage()));\n\n\t\treturn cmd.getExitCodeExceptionMapper() != null ? cmd.getExitCodeExceptionMapper().getExitCode(finalException)\n\t\t\t\t: cmd.getCommandSpec().exitCodeOnExecutionException();\n\t}\n\n\tprivate Throwable unwrapException(Exception exception) {\n\t\tif (exception instanceof RiotException) {\n\t\t\tRiotException riotException = (RiotException) exception;\n\t\t\tif (riotException.getCause() == null) {\n\t\t\t\treturn riotException;\n\t\t\t}\n\t\t\treturn riotException.getCause();\n\t\t}\n\t\treturn exception;\n\t}\n\n}" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/ProcessingItemWriter.java", "content": "package com.redis.riot.core;\n\nimport org.springframework.batch.item.Chunk;\nimport org.springframework.batch.item.ExecutionContext;\nimport org.springframework.batch.item.ItemProcessor;\nimport org.springframework.batch.item.ItemStream;\nimport org.springframework.batch.item.ItemStreamException;\nimport org.springframework.batch.item.ItemStreamSupport;\nimport org.springframework.batch.item.ItemStreamWriter;\nimport org.springframework.batch.item.ItemWriter;\nimport org.springframework.util.ClassUtils;\n\npublic class ProcessingItemWriter extends ItemStreamSupport implements ItemStreamWriter {\n\n\tprivate final ItemProcessor processor;\n\tprivate final ItemWriter writer;\n\n\tpublic ProcessingItemWriter(ItemProcessor processor, ItemWriter writer) {\n\t\tsetName(ClassUtils.getShortName(getClass()));\n\t\tthis.processor = processor;\n\t\tthis.writer = writer;\n\t}\n\n\t@Override\n\tpublic void open(ExecutionContext executionContext) throws ItemStreamException {\n\t\tif (processor instanceof ItemStream) {\n\t\t\t((ItemStream) processor).open(executionContext);\n\n\t\t}\n\t\tif (writer instanceof ItemStream) {\n\t\t\t((ItemStream) writer).open(executionContext);\n\t\t}\n\t}\n\n\t@Override\n\tpublic void close() throws ItemStreamException {\n\t\tif (writer instanceof ItemStream) {\n\t\t\t((ItemStream) writer).close();\n\t\t}\n\t\tif (processor instanceof ItemStream) {\n\t\t\t((ItemStream) processor).close();\n\t\t}\n\t}\n\n\t@Override\n\tpublic void update(ExecutionContext executionContext) throws ItemStreamException {\n\t\tif (processor instanceof ItemStream) {\n\t\t\t((ItemStream) processor).update(executionContext);\n\t\t}\n\t\tif (writer instanceof ItemStream) {\n\t\t\t((ItemStream) writer).update(executionContext);\n\t\t}\n\t}\n\n\t@Override\n\tpublic void write(Chunk chunk) throws Exception {\n\t\tChunk processedChunk = new Chunk<>();\n\t\tfor (S item : chunk) {\n\t\t\tT processedItem = processor.process(item);\n\t\t\tif (processedItem != null) {\n\t\t\t\tprocessedChunk.add(processedItem);\n\t\t\t}\n\t\t}\n\t\twriter.write(processedChunk);\n\t}\n\n}" }, { "path": "core/riot-core/src/main/java/com/redis/riot/core/ProgressArgs.java", "content": "package com.redis.riot.core;\n\nimport lombok.ToString;\nimport picocli.CommandLine.Option;\n\n@ToString\npublic class ProgressArgs {\n\n\tpublic static final RiotDuration DEFAULT_UPDATE_INTERVAL = RiotDuration.ofSeconds(1);\n\tpublic static final ProgressStyle DEFAULT_STYLE = ProgressStyle.ASCII;\n\n\t@Option(names = \"--progress\", description = \"Progress style: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).\", paramLabel = \"