Repository: redis-developer/riot Branch: main Commit: a752f35ab966 Files: 346 Total size: 2.4 MB Directory structure: gitextract_hqj_qnqk/ ├── .github/ │ └── workflows/ │ ├── build.yml │ ├── cache.yml │ ├── early-access.yml │ ├── publish.yml │ ├── release.yml │ ├── step-jlink.yml │ ├── step-precheck.yml │ └── trigger-early-access.yml ├── .gitignore ├── LICENSE ├── README.adoc ├── VERSION ├── build.gradle ├── core/ │ ├── riot-core/ │ │ ├── gradle.properties │ │ ├── riot-core.gradle │ │ └── src/ │ │ ├── main/ │ │ │ ├── java/ │ │ │ │ └── com/ │ │ │ │ └── redis/ │ │ │ │ └── riot/ │ │ │ │ └── core/ │ │ │ │ ├── AbstractCallableCommand.java │ │ │ │ ├── AbstractJobCommand.java │ │ │ │ ├── BaseCommand.java │ │ │ │ ├── CompositeExecutionStrategy.java │ │ │ │ ├── Expression.java │ │ │ │ ├── IO.java │ │ │ │ ├── LoggingMixin.java │ │ │ │ ├── MainCommand.java │ │ │ │ ├── NoopItemWriter.java │ │ │ │ ├── PrintExceptionMessageHandler.java │ │ │ │ ├── ProcessingItemWriter.java │ │ │ │ ├── ProgressArgs.java │ │ │ │ ├── ProgressStepExecutionListener.java │ │ │ │ ├── ProgressStyle.java │ │ │ │ ├── QuietMapAccessor.java │ │ │ │ ├── RetryPolicy.java │ │ │ │ ├── RiotDuration.java │ │ │ │ ├── RiotException.java │ │ │ │ ├── RiotUtils.java │ │ │ │ ├── RiotVersion.java │ │ │ │ ├── SkipPolicy.java │ │ │ │ ├── Step.java │ │ │ │ ├── StepArgs.java │ │ │ │ ├── StepConfiguration.java │ │ │ │ ├── TemplateExpression.java │ │ │ │ ├── ThrottledItemWriter.java │ │ │ │ └── processor/ │ │ │ │ ├── CollectionToMapFunction.java │ │ │ │ ├── ConsumerUnaryOperator.java │ │ │ │ ├── FieldExtractorFactory.java │ │ │ │ ├── FunctionPredicate.java │ │ │ │ ├── IdFunctionBuilder.java │ │ │ │ ├── MapFilteringFunction.java │ │ │ │ ├── MapFlatteningFunction.java │ │ │ │ ├── ObjectToDoubleFunction.java │ │ │ │ ├── ObjectToLongFunction.java │ │ │ │ ├── ObjectToStringFunction.java │ │ │ │ ├── PredicateOperator.java │ │ │ │ ├── RegexNamedGroupFunction.java │ │ │ │ └── StringToMapFunction.java │ │ │ └── resources/ │ │ │ └── com/ │ │ │ └── redis/ │ │ │ └── riot/ │ │ │ └── core/ │ │ │ └── RiotVersion.properties │ │ └── test/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── redis/ │ │ │ └── riot/ │ │ │ └── core/ │ │ │ └── ConverterTests.java │ │ └── resources/ │ │ ├── beers1.csv │ │ └── beers2.csv │ ├── riot-faker/ │ │ ├── gradle.properties │ │ ├── riot-faker.gradle │ │ └── src/ │ │ ├── main/ │ │ │ └── java/ │ │ │ └── com/ │ │ │ └── redis/ │ │ │ └── riot/ │ │ │ └── faker/ │ │ │ └── FakerItemReader.java │ │ └── test/ │ │ └── java/ │ │ └── com/ │ │ └── redis/ │ │ └── riot/ │ │ └── faker/ │ │ └── FakerReaderTests.java │ ├── riot-file/ │ │ ├── gradle.properties │ │ ├── riot-file.gradle │ │ └── src/ │ │ ├── main/ │ │ │ └── java/ │ │ │ └── com/ │ │ │ └── redis/ │ │ │ └── riot/ │ │ │ └── file/ │ │ │ ├── AbstractReaderFactory.java │ │ │ ├── AbstractWriterFactory.java │ │ │ ├── DelimitedReaderFactory.java │ │ │ ├── DelimitedWriterFactory.java │ │ │ ├── FileOptions.java │ │ │ ├── FileReaderRegistry.java │ │ │ ├── FileReaderResult.java │ │ │ ├── FileWriterRegistry.java │ │ │ ├── FileWriterResult.java │ │ │ ├── FixedWidthReaderFactory.java │ │ │ ├── FormattedWriterFactory.java │ │ │ ├── GoogleStorageOptions.java │ │ │ ├── GoogleStorageProtocolResolver.java │ │ │ ├── HeaderCallbackHandler.java │ │ │ ├── JsonLineAggregator.java │ │ │ ├── JsonLinesReaderFactory.java │ │ │ ├── JsonLinesWriterFactory.java │ │ │ ├── JsonReaderFactory.java │ │ │ ├── JsonWriterFactory.java │ │ │ ├── MapFieldSetMapper.java │ │ │ ├── NamedInputStreamResource.java │ │ │ ├── ObjectMapperLineMapper.java │ │ │ ├── OutputStreamResource.java │ │ │ ├── ReadOptions.java │ │ │ ├── ReaderFactory.java │ │ │ ├── ResourceFactory.java │ │ │ ├── ResourceMap.java │ │ │ ├── RiotResourceLoader.java │ │ │ ├── RiotResourceMap.java │ │ │ ├── RuntimeIOException.java │ │ │ ├── S3Options.java │ │ │ ├── S3ProtocolResolver.java │ │ │ ├── StdInProtocolResolver.java │ │ │ ├── StdOutProtocolResolver.java │ │ │ ├── SystemInResource.java │ │ │ ├── SystemOutResource.java │ │ │ ├── UncustomizedUrlResource.java │ │ │ ├── WriteOptions.java │ │ │ ├── WriterFactory.java │ │ │ ├── XmlReaderFactory.java │ │ │ ├── XmlWriterFactory.java │ │ │ └── xml/ │ │ │ ├── XmlItemReader.java │ │ │ ├── XmlItemReaderBuilder.java │ │ │ ├── XmlObjectReader.java │ │ │ ├── XmlResourceItemWriter.java │ │ │ └── XmlResourceItemWriterBuilder.java │ │ └── test/ │ │ └── java/ │ │ └── com/ │ │ └── redis/ │ │ └── riot/ │ │ └── file/ │ │ └── ReaderTests.java │ └── riot-test/ │ ├── gradle.properties │ ├── riot-test.gradle │ └── src/ │ └── main/ │ └── java/ │ └── com/ │ └── redis/ │ └── riot/ │ └── test/ │ └── AbstractRiotTestBase.java ├── docs/ │ ├── LICENSE │ └── guide/ │ ├── gradle.properties │ ├── guide.gradle │ └── src/ │ └── docs/ │ ├── asciidoc/ │ │ ├── _links.adoc │ │ ├── changelog.adoc │ │ ├── concepts.adoc │ │ ├── cookbook.adoc │ │ ├── databases.adoc │ │ ├── datagen.adoc │ │ ├── elasticache.adoc │ │ ├── export.adoc │ │ ├── faq.adoc │ │ ├── files.adoc │ │ ├── import.adoc │ │ ├── index.adoc │ │ ├── install.adoc │ │ ├── introduction.adoc │ │ ├── ping.adoc │ │ ├── replication.adoc │ │ └── usage.adoc │ └── resources/ │ ├── images/ │ │ └── Diagrams.drawio │ ├── order.json │ └── redis-dump.json ├── gradle/ │ └── wrapper/ │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradle.properties ├── gradlew ├── gradlew.bat ├── jreleaser.yml ├── plugins/ │ ├── LICENSE │ └── riot/ │ ├── gradle.properties │ ├── riot.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── redis/ │ │ │ └── riot/ │ │ │ ├── AbstractCompareCommand.java │ │ │ ├── AbstractExportCommand.java │ │ │ ├── AbstractFileExport.java │ │ │ ├── AbstractFileImport.java │ │ │ ├── AbstractImportCommand.java │ │ │ ├── AbstractRedisCommand.java │ │ │ ├── AbstractRedisExportCommand.java │ │ │ ├── AbstractRedisImportCommand.java │ │ │ ├── AbstractRedisTargetExportCommand.java │ │ │ ├── Compare.java │ │ │ ├── CompareLoggingWriteListener.java │ │ │ ├── CompareMode.java │ │ │ ├── CompareStepListener.java │ │ │ ├── ContentType.java │ │ │ ├── DataSourceArgs.java │ │ │ ├── DatabaseExport.java │ │ │ ├── DatabaseImport.java │ │ │ ├── DatabaseReaderArgs.java │ │ │ ├── EvaluationContextArgs.java │ │ │ ├── ExportStepHelper.java │ │ │ ├── ExpressionProcessor.java │ │ │ ├── FakerImport.java │ │ │ ├── FileArgs.java │ │ │ ├── FileExport.java │ │ │ ├── FileImport.java │ │ │ ├── FileReaderArgs.java │ │ │ ├── FileTypeArgs.java │ │ │ ├── FileWriterArgs.java │ │ │ ├── Generate.java │ │ │ ├── GenerateArgs.java │ │ │ ├── GoogleStorageArgs.java │ │ │ ├── ImportProcessorArgs.java │ │ │ ├── JdbcCursorItemReaderFactory.java │ │ │ ├── KeyFilterArgs.java │ │ │ ├── KeyValueDeserializer.java │ │ │ ├── KeyValueFilter.java │ │ │ ├── KeyValueProcessorArgs.java │ │ │ ├── MemoryUsageArgs.java │ │ │ ├── Ping.java │ │ │ ├── PingExecution.java │ │ │ ├── PingExecutionItemReader.java │ │ │ ├── RangeConverter.java │ │ │ ├── ReadFrom.java │ │ │ ├── RedisArgs.java │ │ │ ├── RedisClientArgs.java │ │ │ ├── RedisContext.java │ │ │ ├── RedisReaderArgs.java │ │ │ ├── RedisReaderLiveArgs.java │ │ │ ├── RedisURIConverter.java │ │ │ ├── RedisWriterArgs.java │ │ │ ├── Replicate.java │ │ │ ├── ReplicateReadLogger.java │ │ │ ├── ReplicateWriteLogger.java │ │ │ ├── Riot.java │ │ │ ├── RiotMainCommand.java │ │ │ ├── S3Args.java │ │ │ ├── SourceRedisArgs.java │ │ │ ├── TargetRedisArgs.java │ │ │ ├── ToMapFunction.java │ │ │ ├── Versions.java │ │ │ ├── function/ │ │ │ │ ├── HashToMapFunction.java │ │ │ │ ├── KeyValueMap.java │ │ │ │ ├── MapToFieldFunction.java │ │ │ │ ├── ObjectMapperFunction.java │ │ │ │ ├── StreamItemProcessor.java │ │ │ │ ├── StreamToMapFunction.java │ │ │ │ ├── StringKeyValue.java │ │ │ │ ├── TimeSeriesToMapFunction.java │ │ │ │ ├── ToGeoValue.java │ │ │ │ ├── ToSample.java │ │ │ │ ├── ToScoredValue.java │ │ │ │ ├── ToStringKeyValue.java │ │ │ │ ├── ToSuggestion.java │ │ │ │ └── ZsetToMapFunction.java │ │ │ └── operation/ │ │ │ ├── AbstractMemberOperationCommand.java │ │ │ ├── AbstractOperationCommand.java │ │ │ ├── DelCommand.java │ │ │ ├── ExpireCommand.java │ │ │ ├── ExpireTtlArgs.java │ │ │ ├── FieldFilterArgs.java │ │ │ ├── GeoaddCommand.java │ │ │ ├── HsetCommand.java │ │ │ ├── JsonSetCommand.java │ │ │ ├── LpushCommand.java │ │ │ ├── MemberOperationArgs.java │ │ │ ├── OperationCommand.java │ │ │ ├── RpushCommand.java │ │ │ ├── SaddCommand.java │ │ │ ├── ScoreArgs.java │ │ │ ├── SetCommand.java │ │ │ ├── SugaddCommand.java │ │ │ ├── TsAddCommand.java │ │ │ ├── XaddCommand.java │ │ │ └── ZaddCommand.java │ │ └── resources/ │ │ └── com/ │ │ └── redis/ │ │ └── riot/ │ │ └── banner.txt │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── redis/ │ │ └── riot/ │ │ ├── AbstractRiotApplicationTestBase.java │ │ ├── DbTests.java │ │ ├── FileTests.java │ │ ├── KeyValueSerdeTests.java │ │ ├── PostgresDeltaTests.java │ │ ├── PostgresTests.java │ │ ├── ProcessorTests.java │ │ ├── REServerStack.java │ │ ├── REStack.java │ │ ├── RedisArgsTests.java │ │ ├── RedisContainerFactory.java │ │ ├── RiotTests.java │ │ ├── SqlScriptRunner.java │ │ ├── StackFileTests.java │ │ ├── StackREContainer.java │ │ ├── StackREServer.java │ │ └── StackRiotTests.java │ └── resources/ │ ├── compare-key-processor │ ├── db/ │ │ ├── northwind.sql │ │ ├── oracle.sql │ │ └── postgres-delta.sql │ ├── db-export-postgresql │ ├── db-import-postgresql │ ├── db-import-postgresql-multithreaded │ ├── db-import-postgresql-noop │ ├── db-import-postgresql-set │ ├── faker-hset │ ├── faker-sadd │ ├── faker-tsadd │ ├── faker-tsadd-options │ ├── faker-xadd │ ├── faker-zadd │ ├── file-export-json │ ├── file-export-json-gz │ ├── file-export-xml │ ├── file-import-bad │ ├── file-import-csv │ ├── file-import-csv-max │ ├── file-import-csv-skiplines │ ├── file-import-exclude │ ├── file-import-filetype │ ├── file-import-filter │ ├── file-import-fw │ ├── file-import-gcs │ ├── file-import-geo-processor │ ├── file-import-geoadd │ ├── file-import-hset-expire │ ├── file-import-hset-expire-abs │ ├── file-import-hset-sadd │ ├── file-import-include │ ├── file-import-json │ ├── file-import-json-elastic-hset │ ├── file-import-json-elastic-jsonset │ ├── file-import-json-gz-hset │ ├── file-import-json-hset │ ├── file-import-process │ ├── file-import-process-elvis │ ├── file-import-process-faker │ ├── file-import-process-var │ ├── file-import-psv │ ├── file-import-regex │ ├── file-import-s3 │ ├── file-import-sugadd │ ├── file-import-tsv │ ├── file-import-xml │ ├── files/ │ │ ├── accounts.fw │ │ ├── airports.csv │ │ ├── bad.psv │ │ ├── beers.csv │ │ ├── beers.json │ │ ├── beers.jsonl │ │ ├── beers1.csv │ │ ├── beers2.csv │ │ ├── es_test-index.json │ │ ├── lacity.csv │ │ ├── nobels.json │ │ ├── redis-export.json │ │ ├── redis.json │ │ ├── sample.psv │ │ ├── sample.tsv │ │ ├── timestamp.json │ │ └── trades.xml │ ├── generate │ ├── generate-hash-index │ ├── generate-json-index │ ├── replicate │ ├── replicate-dry-run │ ├── replicate-hll │ ├── replicate-key-exclude │ ├── replicate-key-processor │ ├── replicate-key-processor-compare-none │ ├── replicate-live │ ├── replicate-live-key-exclude │ ├── replicate-live-keyslot │ ├── replicate-live-only-struct │ ├── replicate-live-read-threads │ ├── replicate-live-struct │ ├── replicate-live-threads │ ├── replicate-no-stream-id │ ├── replicate-no-stream-id-prune │ └── replicate-struct ├── riot ├── settings.gradle └── src/ ├── jreleaser/ │ └── changelog.tpl └── media/ └── riot.icns ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/build.yml ================================================ name: Build on: pull_request: permissions: contents: read jobs: build: name: Build runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Setup Java uses: actions/setup-java@v4 with: java-version: 21 distribution: zulu cache: gradle - name: Build shell: bash run: ./gradlew build -S ================================================ FILE: .github/workflows/cache.yml ================================================ name: Clear cache on: schedule: - cron: '0 3 * * *' workflow_dispatch: jobs: clear: name: Delete all caches runs-on: ubuntu-latest steps: - name: Clear caches uses: easimon/wipe-cache@v2 ================================================ FILE: .github/workflows/early-access.yml ================================================ name: EarlyAccess on: push: branches: [ main ] permissions: actions: write id-token: write contents: write jobs: precheck: name: Precheck uses: redis/riot/.github/workflows/step-precheck.yml@main secrets: inherit jlink: name: Jlink needs: [precheck] if: endsWith(${{ needs.precheck.outputs.version }}, '-SNAPSHOT') uses: redis/riot/.github/workflows/step-jlink.yml@main with: project-version: ${{ needs.precheck.outputs.version }} secrets: codecov-token: ${{ secrets.CODECOV_TOKEN }} release: name: Release needs: [precheck, jlink] runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - name: Download artifacts uses: actions/download-artifact@v4 with: name: artifacts path: plugins/riot/build - name: Download repos uses: actions/download-artifact@v4 with: name: repos path: build/repos/local/release - name: Download jlink uses: actions/download-artifact@v4 with: name: jlink path: out/jreleaser/assemble/riot-standalone/jlink - name: Release uses: jreleaser/release-action@v2 with: arguments: full-release env: JRELEASER_DOCKER_DEFAULT_USERNAME: ${{ secrets.DOCKER_USERNAME }} JRELEASER_DOCKER_DEFAULT_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} JRELEASER_GITHUB_PASSWORD: ${{ secrets.GIT_ACCESS_TOKEN }} JRELEASER_GITHUB_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }} JRELEASER_GITHUB_USERNAME: ${{ secrets.GIT_USER }} JRELEASER_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} JRELEASER_GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }} JRELEASER_GPG_SECRET_KEY: ${{ secrets.GPG_SECRET_KEY }} JRELEASER_NEXUS2_USERNAME: ${{ secrets.SONATYPE_USERNAME }} JRELEASER_NEXUS2_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} JRELEASER_PROJECT_VERSION: ${{ needs.precheck.outputs.version }} JRELEASER_SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - name: JReleaser release output if: always() uses: actions/upload-artifact@v4 with: name: riot-release path: | out/jreleaser/trace.log out/jreleaser/output.properties ================================================ FILE: .github/workflows/publish.yml ================================================ name: Publish on: workflow_dispatch: inputs: version: description: 'Release version' required: true permissions: actions: write id-token: write contents: write jobs: precheck: name: Precheck runs-on: ubuntu-latest outputs: version: ${{ steps.vars.outputs.VERSION }} steps: - name: Checkout uses: actions/checkout@v4 publish: name: Publish needs: [precheck] runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: ref: main fetch-depth: 0 - name: Version id: vars shell: bash run: | echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT echo ${{ github.event.inputs.version }} > VERSION - name: Setup Java uses: actions/setup-java@v4 with: java-version: ${{ vars.JAVA_VERSION }} distribution: ${{ vars.JAVA_DISTRO }} cache: gradle - name: Deploy env: GRGIT_USER: ${{ secrets.GIT_USER }} GRGIT_PASS: ${{ secrets.GIT_ACCESS_TOKEN }} run: | ./gradlew -Prelease=true -PreproducibleBuild=true gitPublishPush -S ================================================ FILE: .github/workflows/release.yml ================================================ name: Release on: workflow_dispatch: inputs: version: description: 'Release version' required: true build-tasks: description: 'Build tasks' default: 'build aggregateTestReports aggregateJacocoReport publish' required: false type: string permissions: actions: write id-token: write contents: write jobs: precheck: name: Precheck runs-on: ubuntu-latest outputs: version: ${{ steps.vars.outputs.VERSION }} steps: - name: Checkout uses: actions/checkout@v4 - name: Version id: vars shell: bash run: | echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT echo ${{ github.event.inputs.version }} > VERSION git add VERSION sed -i -e "s/^\:project-version\:\ .*/:project-version: ${{ github.event.inputs.version }}/g" README.adoc git config --global user.email "${{ secrets.COMMIT_EMAIL }}" git config --global user.name "Julien Ruaux" git commit -a -m "Releasing version ${{ github.event.inputs.version }}" git push origin main jlink: name: Jlink needs: [precheck] uses: redis/riot/.github/workflows/step-jlink.yml@main with: project-version: ${{ needs.precheck.outputs.version }} tasks: ${{ github.event.inputs.build-tasks }} secrets: codecov-token: ${{ secrets.CODECOV_TOKEN }} release: name: Release needs: [precheck, jlink] runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: ref: main fetch-depth: 0 - name: Download artifacts uses: actions/download-artifact@v4 with: name: artifacts path: plugins/riot/build - name: Download repos uses: actions/download-artifact@v4 with: name: repos path: build/repos/local/release - name: Download jlink uses: actions/download-artifact@v4 with: name: jlink path: out/jreleaser/assemble/riot-standalone/jlink - name: Setup Java uses: actions/setup-java@v4 with: java-version: ${{ vars.JAVA_VERSION }} distribution: ${{ vars.JAVA_DISTRO }} cache: gradle - name: Deploy env: GRGIT_USER: ${{ secrets.GIT_USER }} GRGIT_PASS: ${{ secrets.GIT_ACCESS_TOKEN }} run: | ./gradlew -Prelease=true -PreproducibleBuild=true publish gitPublishPush -S - name: Upload deploy artifacts uses: actions/upload-artifact@v4 with: retention-days: 7 name: deploy path: | build/repos/local/release/ - name: Release uses: jreleaser/release-action@v2 with: arguments: full-release env: JRELEASER_DOCKER_DEFAULT_USERNAME: ${{ secrets.DOCKER_USERNAME }} JRELEASER_DOCKER_DEFAULT_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} JRELEASER_GITHUB_PASSWORD: ${{ secrets.GIT_ACCESS_TOKEN }} JRELEASER_GITHUB_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }} JRELEASER_GITHUB_USERNAME: ${{ secrets.GIT_USER }} JRELEASER_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} JRELEASER_GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }} JRELEASER_GPG_SECRET_KEY: ${{ secrets.GPG_SECRET_KEY }} JRELEASER_NEXUS2_USERNAME: ${{ secrets.SONATYPE_USERNAME }} JRELEASER_NEXUS2_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} JRELEASER_PROJECT_VERSION: ${{ needs.precheck.outputs.version }} JRELEASER_SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - name: JReleaser release output if: always() uses: actions/upload-artifact@v4 with: name: riot-release path: | out/jreleaser/trace.log out/jreleaser/output.properties ================================================ FILE: .github/workflows/step-jlink.yml ================================================ name: X-Jlink on: workflow_call: inputs: project-version: required: true type: string tasks: default: 'build aggregateTestReports aggregateJacocoReport publish' required: false type: string secrets: codecov-token: required: false permissions: contents: read jobs: jlink: name: Jlink runs-on: ubuntu-latest env: CI: true steps: - name: Checkout uses: actions/checkout@v4 with: ref: main - name: Setup Java uses: actions/setup-java@v4 with: java-version: ${{ vars.JAVA_VERSION }} distribution: ${{ vars.JAVA_DISTRO }} cache: gradle - name: Version if: endsWith(${{ inputs.project-version }}, '-SNAPSHOT') != true run: | echo ${{ inputs.project-version }} > VERSION - name: Build run: ./gradlew -Prelease=true -PreproducibleBuild=true ${{ inputs.tasks }} -S - name: Upload test reports if: failure() uses: actions/upload-artifact@v4 with: name: test-reports path: | build/reports/aggregate-tests/ - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 env: CODECOV_TOKEN: ${{ secrets.codecov-token }} - name: Jlink uses: jreleaser/release-action@v2 with: arguments: assemble --assembler jlink setup-java: false env: JRELEASER_PROJECT_VERSION: ${{ inputs.project-version }} - name: JReleaser output if: always() uses: actions/upload-artifact@v4 with: name: riot-jlink path: | out/jreleaser/trace.log out/jreleaser/output.properties - name: Dependencies run: | ls -l plugins/riot/build/dependencies/flat - name: Upload repos uses: actions/upload-artifact@v4 with: retention-days: 1 name: repos path: | build/repos/local/release - name: Upload artifacts uses: actions/upload-artifact@v4 with: retention-days: 1 name: artifacts path: | plugins/riot/build/libs/ plugins/riot/build/dependencies/ plugins/riot/build/distributions/ - name: Stop Gradle daemon shell: bash run: ./gradlew -stop - name: Upload jlink uses: actions/upload-artifact@v4 with: retention-days: 1 name: jlink path: | out/jreleaser/assemble/riot-standalone/jlink/*.zip ================================================ FILE: .github/workflows/step-precheck.yml ================================================ name: X-Precheck on: workflow_call: outputs: version: description: version value: ${{ jobs.precheck.outputs.version }} permissions: contents: read jobs: precheck: name: Precheck if: github.repository == 'redis/riot' && startsWith(github.event.head_commit.message, 'Releasing version') != true runs-on: ubuntu-latest outputs: version: ${{ steps.vars.outputs.VERSION }} steps: - name: Checkout uses: actions/checkout@v4 - name: Cancel previous run uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ secrets.GITHUB_TOKEN }} - name: Version id: vars shell: bash run: | echo "VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT ================================================ FILE: .github/workflows/trigger-early-access.yml ================================================ name: Trigger Early Access on: workflow_dispatch: permissions: contents: read jobs: earlyaccess: name: Trigger Early Access runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Setup Java uses: actions/setup-java@v4 with: java-version: ${{ vars.JAVA_VERSION }} distribution: ${{ vars.JAVA_DISTRO }} cache: gradle - name: Build run: ./gradlew -PreproducibleBuild=true build -S - name: Rename artifacts run: | mkdir early-access cp plugins/riot/build/distributions/riot-*-SNAPSHOT.zip early-access/riot-early-access.zip - name: Release early-access artifacts uses: marvinpinto/action-automatic-releases@latest with: automatic_release_tag: early-access repo_token: ${{ secrets.GIT_ACCESS_TOKEN }} prerelease: true title: RIOT Early-Access files: | early-access/* ================================================ FILE: .gitignore ================================================ /out/ build .gradle /.idea bin .project .classpath .settings .DS_Store hs_err_pid*.log *.bkp ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS ================================================ FILE: README.adoc ================================================ :linkattrs: :project-owner: redis :project-name: riot :project-group: com.redis :project-version: 4.3.0 :project-title: RIOT image::docs/guide/src/docs/resources/images/riot.svg[RIOT] --- [IMPORTANT] ==== *⚠️ This project is no longer maintained.* RIOT has been superseded by https://github.com/redis/riotx-dist[*RIOT-X*], an enterprise-grade version with enhanced features and active support from Redis, Inc. * *For new projects:* Please use https://redis.github.io/riotx/[RIOT-X] * *Existing users:* You can continue using RIOT, but no new features or bug fixes will be provided * *Support:* All support requests should be directed to the RIOT-X project https://github.com/redis/riotx-dist[→ Go to RIOT-X on GitHub] ==== --- Get data in and out of Redis with link:http://redis.github.io/riot/[{project-title}]! Redis Input/Output Tools ({project-title}) is a command-line utility designed to help you get data in and out of Redis. It supports many different sources and targets: * Relational databases * Files (CSV, JSON, XML) * Data generators (Redis data structures, Faker) * and Redis itself with live and/or snapshot replication. == Documentation link:http://redis.github.io/riot/[redis.github.io/riot] == Support *This project is no longer actively maintained.* For support, please use https://github.com/redis/riotx-dist[RIOT-X], the actively maintained successor to this project. Existing issues will remain open for historical reference, but new issues and pull requests will not be addressed. == Badges image:https://github.com/{project-owner}/{project-name}/actions/workflows/early-access.yml/badge.svg["Build Status", link="https://github.com/{project-owner}/{project-name}/actions/workflows/early-access.yml"] image:https://img.shields.io/github/release/{project-owner}/{project-name}.svg["Latest", link="https://github.com/{project-owner}/{project-name}/releases/latest"] image:https://codecov.io/gh/{project-owner}/{project-name}/graph/badge.svg?token=bwm6gyXU0v["Coverage", link="https://codecov.io/gh/{project-owner}/{project-name}"] == License {project-title} is licensed under the Apache 2.0 License. Copyright (C) 2023 Redis, Inc. ================================================ FILE: VERSION ================================================ 4.3.0 ================================================ FILE: build.gradle ================================================ /* * SPDX-License-Identifier: Apache-2.0 * * Copyright 2020-2023 The RIOT authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ config { info { description = 'RIOT' inceptionYear = '2020' vendor = 'Redis' tags = ['redis', 'tool', 'import', 'export', 'replication'] links { website = "https://github.com/redis/${project.rootProject.name}" issueTracker = "https://github.com/redis/${project.rootProject.name}/issues" scm = "https://github.com/redis/${project.rootProject.name}.git" } scm { url = "https://github.com/redis/${project.rootProject.name}" connection = "scm:git:https://github.com/redis/${project.rootProject.name}.git" developerConnection = "scm:git:git@github.com:redis/${project.rootProject.name}.git" } specification { enabled = true } implementation { enabled = true } people { clearDomainSet() person { id = 'jruaux' name = 'Julien Ruaux' roles = ['developer', 'author'] } } } docs { javadoc { autoLinks { enabled = false } } } licensing { enabled = false } coverage { jacoco { enabled = true toolVersion = jacocoPluginVersion } } } allprojects { repositories { mavenLocal() maven { url 'https://s01.oss.sonatype.org/content/repositories/snapshots/' } } tasks.withType(GenerateModuleMetadata) { enabled = false } tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } } subprojects { subproj -> if (!subproj.name.contains('guide')) { config { info { description = project.project_description } coverage { jacoco { toolVersion = jacocoPluginVersion } } } license { exclude('build/**') } dependencies { compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.junit.jupiter:junit-jupiter-api' testImplementation 'org.junit.jupiter:junit-jupiter-params' testImplementation 'org.junit.jupiter:junit-jupiter-engine' testImplementation 'org.junit.platform:junit-platform-launcher' testImplementation 'org.testcontainers:junit-jupiter' testImplementation group: 'com.redis', name: 'testcontainers-redis', version: testcontainersRedisVersion } test { useJUnitPlatform() } configurations { all*.exclude module: 'spring-boot-starter-logging' all*.exclude module: 'commons-logging' } bootJar { enabled = false } jar { enabled = true archiveClassifier = '' } } } subprojects { p -> config { coverage { jacoco { enabled = hasTestsAt(p.file('src/test')) } } } configurations.all { resolutionStrategy.eachDependency { DependencyResolveDetails details -> if (details.requested.name == 'lettuce-core' ) { details.useVersion lettuceVersion } } } } gradleProjects { subprojects { dirs(['core']) { compileJava { options.compilerArgs += ["-AprojectPath=${project.group}/${project.name}"] } if (!(project.findProperty('automatic.module.name.skip') ?: false).toBoolean()) { jar { manifest { attributes('Automatic-Module-Name': project.findProperty('automatic.module.name')) } } } } } } boolean hasTestsAt(File testDir) { testDir.exists() && testDir.listFiles()?.size() } ================================================ FILE: core/riot-core/gradle.properties ================================================ # # SPDX-License-Identifier: Apache-2.0 # # Copyright 2022-2023 The RIOT authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # project_description = RIOT Core automatic.module.name = com.redis.riot.core ================================================ FILE: core/riot-core/riot-core.gradle ================================================ /* * SPDX-License-Identifier: Apache-2.0 * * Copyright 2020-2023 The RIOT authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ dependencies { api 'org.slf4j:slf4j-api' implementation 'org.slf4j:slf4j-simple' api group: 'info.picocli', name: 'picocli', version: picocliVersion annotationProcessor group: 'info.picocli', name: 'picocli-codegen', version: picocliVersion implementation 'org.springframework.boot:spring-boot' implementation group: 'me.tongfei', name: 'progressbar', version: progressbarVersion api group: 'com.redis', name: 'spring-batch-redis-infrastructure', version: springBatchRedisVersion testImplementation 'org.awaitility:awaitility' } project.rootProject.gradle.addBuildListener(new BuildAdapter() { @Override void projectsEvaluated(Gradle gradle) { gradle.rootProject.subprojects .find { p -> p.name == 'riot-core' } .processResources { inputs.property('build_date', gradle.rootProject.config.buildInfo.buildDate + ':' + gradle.rootProject.config.buildInfo.buildTime) filesMatching(['**/RiotVersion.properties']) { expand( 'riot_version': gradle.rootProject.version, 'build_date': gradle.rootProject.config.buildInfo.buildDate, 'build_time': gradle.rootProject.config.buildInfo.buildTime, 'build_revision': gradle.rootProject.config.buildInfo.buildRevision ) } } } }) ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/AbstractCallableCommand.java ================================================ package com.redis.riot.core; import java.util.concurrent.Callable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @Command public abstract class AbstractCallableCommand extends BaseCommand implements Callable { @Option(names = "--help", usageHelp = true, description = "Show this help message and exit.") private boolean helpRequested; protected Logger log; @Override public Integer call() throws Exception { initialize(); try { execute(); } finally { teardown(); } return 0; } protected void initialize() { if (log == null) { log = LoggerFactory.getLogger(getClass()); } } protected abstract void execute(); protected void teardown() { // do nothing } public Logger getLog() { return log; } public void setLog(Logger log) { this.log = log; } } ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/AbstractJobCommand.java ================================================ package com.redis.riot.core; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.List; import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.ItemWriteListener; import org.springframework.batch.core.Job; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobExecutionException; import org.springframework.batch.core.JobExecutionListener; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; import org.springframework.batch.core.JobParametersInvalidException; import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.job.builder.SimpleJobBuilder; import org.springframework.batch.core.launch.JobLauncher; import org.springframework.batch.core.launch.support.RunIdIncrementer; import org.springframework.batch.core.launch.support.TaskExecutorJobLauncher; import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.repository.JobRestartException; import org.springframework.batch.core.step.builder.FaultTolerantStepBuilder; import org.springframework.batch.core.step.builder.SimpleStepBuilder; import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.core.step.skip.AlwaysSkipItemSkipPolicy; import org.springframework.batch.core.step.skip.NeverSkipItemSkipPolicy; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemStreamSupport; import org.springframework.batch.item.ItemWriter; import org.springframework.batch.item.support.SynchronizedItemReader; import org.springframework.batch.item.support.SynchronizedItemStreamReader; import org.springframework.core.task.SyncTaskExecutor; import org.springframework.core.task.TaskExecutor; import org.springframework.retry.policy.AlwaysRetryPolicy; import org.springframework.retry.policy.NeverRetryPolicy; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.util.Assert; import org.springframework.util.ClassUtils; import org.springframework.util.CollectionUtils; import com.redis.spring.batch.JobUtils; import com.redis.spring.batch.item.AbstractAsyncItemStreamSupport; import com.redis.spring.batch.item.PollableItemReader; import com.redis.spring.batch.step.FlushingStepBuilder; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @Command public abstract class AbstractJobCommand extends AbstractCallableCommand { public static final String DEFAULT_JOB_REPOSITORY_NAME = "riot"; @Option(names = "--job-name", description = "Job name.", paramLabel = "", hidden = true) private String jobName; @Option(names = "--repeat", description = "After the job completes keep repeating it on a fixed interval (ex 5m, 1h)", paramLabel = "") private RiotDuration repeatEvery; @ArgGroup(exclusive = false, heading = "Job options%n") private StepArgs stepArgs = new StepArgs(); private String jobRepositoryName = DEFAULT_JOB_REPOSITORY_NAME; private JobRepository jobRepository; private PlatformTransactionManager transactionManager; private JobLauncher jobLauncher; private JobExplorer jobExplorer; protected Runnable onJobSuccessCallback; @Override protected void initialize() { super.initialize(); if (jobName == null) { jobName = jobName(); } if (jobRepository == null) { try { jobRepository = JobUtils.jobRepositoryFactoryBean(jobRepositoryName).getObject(); } catch (Exception e) { throw new RiotException("Could not create job repository", e); } } if (transactionManager == null) { transactionManager = JobUtils.resourcelessTransactionManager(); } if (jobLauncher == null) { try { jobLauncher = jobLauncher(); } catch (Exception e) { throw new RiotException("Could not create job launcher", e); } } if (jobExplorer == null) { try { jobExplorer = JobUtils.jobExplorerFactoryBean(jobRepositoryName).getObject(); } catch (Exception e) { log.warn("Error getting jobExplorer", e); throw new RiotException("Could not create job explorer", e); } } } private JobLauncher jobLauncher() throws Exception { TaskExecutorJobLauncher launcher = new TaskExecutorJobLauncher(); launcher.setJobRepository(jobRepository); launcher.setTaskExecutor(new SyncTaskExecutor()); launcher.afterPropertiesSet(); return launcher; } protected void configureAsyncStreamSupport(AbstractAsyncItemStreamSupport reader) { reader.setJobRepository(jobRepository); } private JobBuilder jobBuilder() { return new JobBuilder(jobName, jobRepository); } @Override protected void execute() { Job job = job(); JobExecution jobExecution; try { jobExecution = jobLauncher.run(job, new JobParameters()); } catch (JobExecutionException e) { throw new RiotException(e); } if (JobUtils.isFailed(jobExecution.getExitStatus())) { for (StepExecution stepExecution : jobExecution.getStepExecutions()) { ExitStatus stepExitStatus = stepExecution.getExitStatus(); if (JobUtils.isFailed(stepExitStatus)) { if (CollectionUtils.isEmpty(stepExecution.getFailureExceptions())) { throw new RiotException(stepExitStatus.getExitDescription()); } throw wrapException(stepExecution.getFailureExceptions()); } } throw wrapException(jobExecution.getFailureExceptions()); } } private String jobName() { if (commandSpec == null) { return ClassUtils.getShortName(getClass()); } return commandSpec.name(); } private RiotException wrapException(List throwables) { if (throwables.isEmpty()) { return new RiotException("Job failed"); } return new RiotException(throwables.get(0)); } protected Job job(Step... steps) { return job(Arrays.asList(steps)); } protected Job job(Collection> steps) { Assert.notEmpty(steps, "At least one step must be specified"); Iterator> iterator = steps.iterator(); SimpleJobBuilder job = jobBuilder().start(step(iterator.next())); while (iterator.hasNext()) { job.next(step(iterator.next())); } if (repeatEvery != null) { job.incrementer(new RunIdIncrementer()); job.preventRestart(); job.listener(new RepeatJobExecutionListener(job, steps)); } return job.build(); } private class RepeatJobExecutionListener implements JobExecutionListener { private final SimpleJobBuilder job; private final Collection> steps; private Job lastJob; public RepeatJobExecutionListener(SimpleJobBuilder job, Collection> steps) { this.job = job; this.steps = steps; } @Override public void afterJob(JobExecution jobExecution) { if (jobExecution.getStatus() == BatchStatus.COMPLETED) { if (null != onJobSuccessCallback) { onJobSuccessCallback.run(); } log.info("Finished job, will run again in {}", repeatEvery); try { Thread.sleep(repeatEvery.getValue().toMillis()); if (lastJob == null) { lastJob = job.build(); } Job nextJob = jobBuilder().start(step(steps.stream().findFirst().get())) .incrementer(new RunIdIncrementer()).preventRestart().listener(this).build(); JobParametersBuilder paramsBuilder = new JobParametersBuilder(jobExecution.getJobParameters(), jobExplorer); jobLauncher.run(nextJob, paramsBuilder.addString("runTime", String.valueOf(System.currentTimeMillis())) .getNextJobParameters(lastJob).toJobParameters()); lastJob = nextJob; } catch (InterruptedException | JobExecutionAlreadyRunningException | JobRestartException | JobInstanceAlreadyCompleteException | JobParametersInvalidException e) { throw new RiotException(e); } } JobExecutionListener.super.afterJob(jobExecution); } } protected boolean shouldShowProgress() { return stepArgs.getProgressArgs().getStyle() != ProgressStyle.NONE; } protected abstract Job job(); private TaskletStep step(Step step) { log.info("Creating {}", step); SimpleStepBuilder builder = simpleStep(step); if (stepArgs.getRetryPolicy() == RetryPolicy.NEVER && stepArgs.getSkipPolicy() == SkipPolicy.NEVER) { log.info("Skipping fault-tolerance for step {}", step.getName()); return builder.build(); } log.info("Adding fault-tolerance to step {}", step.getName()); FaultTolerantStepBuilder ftStep = builder.faultTolerant(); step.getSkip().forEach(ftStep::skip); step.getNoSkip().forEach(ftStep::noSkip); step.getRetry().forEach(ftStep::retry); step.getNoRetry().forEach(ftStep::noRetry); ftStep.retryLimit(stepArgs.getRetryLimit()); ftStep.retryPolicy(retryPolicy()); ftStep.skipLimit(stepArgs.getSkipLimit()); ftStep.skipPolicy(skipPolicy()); return ftStep.build(); } private org.springframework.retry.RetryPolicy retryPolicy() { switch (stepArgs.getRetryPolicy()) { case ALWAYS: return new AlwaysRetryPolicy(); case NEVER: return new NeverRetryPolicy(); default: return null; } } private org.springframework.batch.core.step.skip.SkipPolicy skipPolicy() { switch (stepArgs.getSkipPolicy()) { case ALWAYS: return new AlwaysSkipItemSkipPolicy(); case NEVER: return new NeverSkipItemSkipPolicy(); default: return null; } } @SuppressWarnings("removal") private SimpleStepBuilder simpleStep(Step step) { String stepName = jobName + "-" + step.getName(); if (stepName.length() > 80) { stepName = stepName.substring(0, 69) + "…" + stepName.substring(stepName.length() - 10); } if (step.getReader() instanceof ItemStreamSupport) { ((ItemStreamSupport) step.getReader()).setName(stepName + "-reader"); } log.info("Creating step {} with chunk size {}", stepName, stepArgs.getChunkSize()); SimpleStepBuilder builder = new StepBuilder(stepName, jobRepository).chunk(stepArgs.getChunkSize(), transactionManager); builder.reader(reader(step)); builder.writer(writer(step)); builder.processor(step.getProcessor()); builder.taskExecutor(taskExecutor()); builder.throttleLimit(stepArgs.getThreads()); step.getExecutionListeners().forEach(builder::listener); step.getWriteListeners().forEach(builder::listener); if (shouldShowProgress()) { ProgressStepExecutionListener listener = new ProgressStepExecutionListener<>(step); builder.listener((StepExecutionListener) listener); builder.listener((ItemWriteListener) listener); } if (step.isLive()) { log.info("Creating flushing step with flush interval {} and idle timeout {}", step.getFlushInterval(), step.getIdleTimeout()); FlushingStepBuilder flushingStepBuilder = new FlushingStepBuilder<>(builder); flushingStepBuilder.flushInterval(step.getFlushInterval()); flushingStepBuilder.idleTimeout(step.getIdleTimeout()); return flushingStepBuilder; } return builder; } private TaskExecutor taskExecutor() { if (stepArgs.getThreads() == 1) { return new SyncTaskExecutor(); } log.info("Creating thread-pool task executor of size {}", stepArgs.getThreads()); ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); taskExecutor.setMaxPoolSize(stepArgs.getThreads()); taskExecutor.setCorePoolSize(stepArgs.getThreads()); taskExecutor.initialize(); return taskExecutor; } private ItemReader reader(Step step) { if (stepArgs.getThreads() == 1 || step.getReader() instanceof PollableItemReader) { return step.getReader(); } log.info("Synchronizing reader in step {}", step.getName()); if (step.getReader() instanceof ItemStreamReader) { SynchronizedItemStreamReader synchronizedReader = new SynchronizedItemStreamReader<>(); synchronizedReader.setDelegate((ItemStreamReader) step.getReader()); return synchronizedReader; } return new SynchronizedItemReader<>(step.getReader()); } private ItemWriter writer(Step step) { ItemWriter writer = step.getWriter(); if (stepArgs.isDryRun()) { log.info("Using no-op writer"); writer = new NoopItemWriter<>(); } if (stepArgs.getSleep() != null) { log.info("Throttling writer with sleep={}", stepArgs.getSleep()); writer = new ThrottledItemWriter<>(writer, stepArgs.getSleep().getValue()); } return writer; } public String getJobName() { return jobName; } public void setJobName(String name) { this.jobName = name; } public StepArgs getJobArgs() { return stepArgs; } public void setJobArgs(StepArgs args) { this.stepArgs = args; } public String getJobRepositoryName() { return jobRepositoryName; } public void setJobRepositoryName(String jobRepositoryName) { this.jobRepositoryName = jobRepositoryName; } public JobRepository getJobRepository() { return jobRepository; } public void setJobRepository(JobRepository jobRepository) { this.jobRepository = jobRepository; } public PlatformTransactionManager getTransactionManager() { return transactionManager; } public void setTransactionManager(PlatformTransactionManager transactionManager) { this.transactionManager = transactionManager; } public JobLauncher getJobLauncher() { return jobLauncher; } public void setJobLauncher(JobLauncher jobLauncher) { this.jobLauncher = jobLauncher; } public JobExplorer getJobExplorer() { return jobExplorer; } public void setJobExplorer(JobExplorer jobExplorer) { this.jobExplorer = jobExplorer; } } ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/BaseCommand.java ================================================ package com.redis.riot.core; import java.util.Map; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Option; import picocli.CommandLine.Spec; @Command(usageHelpAutoWidth = true, mixinStandardHelpOptions = true, abbreviateSynopsis = true) public class BaseCommand { static { if (System.getenv().containsKey("RIOT_NO_COLOR")) { System.setProperty("picocli.ansi", "false"); } } @Spec protected CommandSpec commandSpec; @Mixin LoggingMixin loggingMixin; @Option(names = "-D", paramLabel = "", description = "Sets a System property.", mapFallbackValue = "", hidden = true) void setProperty(Map props) { props.forEach(System::setProperty); } } ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/CompositeExecutionStrategy.java ================================================ package com.redis.riot.core; import picocli.CommandLine.ExecutionException; import picocli.CommandLine.IExecutionStrategy; import picocli.CommandLine.ParameterException; import picocli.CommandLine.ParseResult; public class CompositeExecutionStrategy implements IExecutionStrategy { private final IExecutionStrategy inactive; private final IExecutionStrategy active; public CompositeExecutionStrategy(IExecutionStrategy inactive, IExecutionStrategy active) { this.inactive = inactive; this.active = active; } @Override public int execute(ParseResult parseResult) throws ExecutionException, ParameterException { inactive.execute(parseResult); return active.execute(parseResult); } } ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/Expression.java ================================================ package com.redis.riot.core; import java.util.function.Predicate; import org.springframework.expression.EvaluationContext; import org.springframework.expression.common.TemplateParserContext; import org.springframework.expression.spel.standard.SpelExpressionParser; public class Expression { protected static final SpelExpressionParser PARSER = new SpelExpressionParser(); protected final org.springframework.expression.Expression spelExpression; public Expression(org.springframework.expression.Expression expression) { this.spelExpression = expression; } @Override public String toString() { return spelExpression.getExpressionString(); } public Predicate predicate(EvaluationContext context) { return t -> spelExpression.getValue(context, t, Boolean.class); } public Object getValue(EvaluationContext context) { return spelExpression.getValue(context); } public Object getValue(EvaluationContext context, Object rootObject) { return spelExpression.getValue(context, rootObject); } public Long getLong(EvaluationContext context, Object rootObject) { return spelExpression.getValue(context, rootObject, Long.class); } public String getString(EvaluationContext context, Object rootObject) { return spelExpression.getValue(context, rootObject, String.class); } public static Expression parse(String expression) { return new Expression(PARSER.parseExpression(expression)); } public static TemplateExpression parseTemplate(String expression) { return new TemplateExpression(PARSER.parseExpression(expression, new TemplateParserContext())); } } ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/IO.java ================================================ package com.redis.riot.core; import java.io.PrintWriter; public interface IO { PrintWriter getOut(); void setOut(PrintWriter out); PrintWriter getErr(); void setErr(PrintWriter err); } ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/LoggingMixin.java ================================================ package com.redis.riot.core; import static picocli.CommandLine.Spec.Target.MIXEE; import java.util.LinkedHashMap; import java.util.Map; import org.slf4j.event.Level; import org.slf4j.simple.SimpleLogger; import picocli.CommandLine.Model.CommandSpec; import picocli.CommandLine.Option; import picocli.CommandLine.ParseResult; import picocli.CommandLine.Spec; public class LoggingMixin { public static final boolean DEFAULT_SHOW_DATE_TIME = true; public static final boolean DEFAULT_SHOW_THREAD_NAME = true; public static final boolean DEFAULT_SHOW_LOG_NAME = true; public static final Level DEFAULT_LEVEL = Level.WARN; /** * This mixin is able to climb the command hierarchy because the * {@code @Spec(Target.MIXEE)}-annotated field gets a reference to the command * where it is used. */ private @Spec(MIXEE) CommandSpec mixee; // spec of the command where the @Mixin is used private String file; private boolean showDateTime = DEFAULT_SHOW_DATE_TIME; private String dateTimeFormat; private boolean showThreadId; private boolean showThreadName = DEFAULT_SHOW_THREAD_NAME; private boolean showLogName = DEFAULT_SHOW_LOG_NAME; private boolean showShortLogName; private boolean levelInBrackets; private Map levels = new LinkedHashMap<>(); private Level level = DEFAULT_LEVEL; private static LoggingMixin getTopLevelCommandLoggingMixin(CommandSpec commandSpec) { return ((MainCommand) commandSpec.root().userObject()).loggingMixin; } @Option(names = "--log-file", description = "Log output target. Can be a path or special values System.out and System.err (default: System.err).", paramLabel = "") public void setFile(String file) { getTopLevelCommandLoggingMixin(mixee).file = file; } @Option(names = "--log-time", description = "Include current date and time in log messages. True by default.", negatable = true, defaultValue = "true", fallbackValue = "true") public void setShowDateTime(boolean show) { getTopLevelCommandLoggingMixin(mixee).showDateTime = show; } @Option(names = "--log-time-fmt", defaultValue = "yyyy-MM-dd HH:mm:ss.SSS", description = "Date and time format to be used in log messages (default: ${DEFAULT-VALUE}). Use with --log-time.", paramLabel = "") public void setDateTimeFormat(String format) { getTopLevelCommandLoggingMixin(mixee).dateTimeFormat = format; } @Option(names = "--log-thread-id", description = "Include current thread ID in log messages.", hidden = true) public void setShowThreadId(boolean show) { getTopLevelCommandLoggingMixin(mixee).showThreadId = show; } @Option(names = "--log-thread", description = "Show current thread name in log messages. True by default.", negatable = true, defaultValue = "true", fallbackValue = "true", hidden = true) public void setShowThreadName(boolean show) { getTopLevelCommandLoggingMixin(mixee).showThreadName = show; } @Option(names = "--log-name", description = "Show logger instance name in log messages. True by default.", negatable = true, defaultValue = "true", fallbackValue = "true", hidden = true) public void setShowLogName(boolean show) { getTopLevelCommandLoggingMixin(mixee).showLogName = show; } @Option(names = "--log-short", description = "Include last component of logger instance name in log messages. True by default.", negatable = true, defaultValue = "true", fallbackValue = "true", hidden = true) public void setShowShortLogName(boolean show) { getTopLevelCommandLoggingMixin(mixee).showShortLogName = show; } @Option(names = "--log-level-brck", description = "Output log level string in brackets.", hidden = true) public void setLevelInBrackets(boolean enable) { getTopLevelCommandLoggingMixin(mixee).levelInBrackets = enable; } @Option(arity = "1..*", names = "--log", description = "Custom log levels (e.g.: io.lettuce=INFO).", paramLabel = "") public void setLevels(Map levels) { getTopLevelCommandLoggingMixin(mixee).levels = levels; } @Option(names = { "-d", "--debug" }, description = "Log in debug mode.") public void setDebug(boolean enable) { if (enable) { getTopLevelCommandLoggingMixin(mixee).level = Level.DEBUG; } } @Option(names = { "-i", "--info" }, description = "Set log level to info.") public void setInfo(boolean enable) { if (enable) { getTopLevelCommandLoggingMixin(mixee).level = Level.INFO; } } @Option(names = { "-q", "--quiet" }, description = "Log errors only.") public void setError(boolean enable) { if (enable) { getTopLevelCommandLoggingMixin(mixee).level = Level.ERROR; } } @Option(names = "--log-level", description = "Set log level: ${COMPLETION-CANDIDATES} (default: WARN).", paramLabel = "", hidden = true) public void setLevel(Level level) { getTopLevelCommandLoggingMixin(mixee).level = level; } public static int executionStrategy(ParseResult parseResult) { getTopLevelCommandLoggingMixin(parseResult.commandSpec()).configureLoggers(); return 0; } public void configureLoggers() { LoggingMixin mixin = getTopLevelCommandLoggingMixin(mixee); System.setProperty(SimpleLogger.DEFAULT_LOG_LEVEL_KEY, mixin.level.name()); if (mixin.file != null) { System.setProperty(SimpleLogger.LOG_FILE_KEY, mixin.file); } setBoolean(SimpleLogger.SHOW_DATE_TIME_KEY, mixin.showDateTime); if (mixin.dateTimeFormat != null) { System.setProperty(SimpleLogger.DATE_TIME_FORMAT_KEY, mixin.dateTimeFormat); } setBoolean(SimpleLogger.SHOW_THREAD_ID_KEY, mixin.showThreadId); setBoolean(SimpleLogger.SHOW_THREAD_NAME_KEY, mixin.showThreadName); setBoolean(SimpleLogger.SHOW_LOG_NAME_KEY, mixin.showLogName); setBoolean(SimpleLogger.SHOW_SHORT_LOG_NAME_KEY, mixin.showShortLogName); setBoolean(SimpleLogger.LEVEL_IN_BRACKETS_KEY, mixin.levelInBrackets); setLogLevel("com.amazonaws.internal", Level.ERROR); setLogLevel("com.redis.spring.batch.step.FlushingFaultTolerantStepBuilder", Level.ERROR); setLogLevel("org.springframework.batch.core.step.builder.FaultTolerantStepBuilder", Level.ERROR); setLogLevel("org.springframework.batch.core.step.item.ChunkMonitor", Level.ERROR); mixin.levels.forEach(this::setLogLevel); } private void setLogLevel(String key, Level level) { System.setProperty(SimpleLogger.LOG_KEY_PREFIX + key, level.name()); } private void setBoolean(String property, boolean value) { System.setProperty(property, String.valueOf(value)); } public boolean isStacktrace() { return level.toInt() <= Level.INFO.toInt(); } } ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/MainCommand.java ================================================ package com.redis.riot.core; import java.io.PrintWriter; import java.util.concurrent.Callable; import org.springframework.util.unit.DataSize; import picocli.CommandLine; import picocli.CommandLine.ParseResult; import picocli.CommandLine.RunLast; public class MainCommand extends BaseCommand implements Callable, IO { private PrintWriter out; private PrintWriter err; @Override public Integer call() throws Exception { commandSpec.commandLine().usage(out); return 0; } protected CommandLine commandLine() { return new CommandLine(this); } public int run(String... args) { CommandLine commandLine = commandLine(); setOut(commandLine.getOut()); setErr(commandLine.getErr()); commandLine.setCaseInsensitiveEnumValuesAllowed(true); commandLine.setUnmatchedOptionsAllowedAsOptionParameters(false); commandLine.setExecutionExceptionHandler(new PrintExceptionMessageHandler()); registerConverters(commandLine); commandLine.setExecutionStrategy( new CompositeExecutionStrategy(LoggingMixin::executionStrategy, this::executionStrategy)); return commandLine.execute(args); } protected int executionStrategy(ParseResult parseResult) { return new RunLast().execute(parseResult); } protected void registerConverters(CommandLine commandLine) { commandLine.registerConverter(RiotDuration.class, RiotDuration::parse); commandLine.registerConverter(DataSize.class, MainCommand::parseDataSize); commandLine.registerConverter(Expression.class, Expression::parse); commandLine.registerConverter(TemplateExpression.class, Expression::parseTemplate); } public static DataSize parseDataSize(String string) { return DataSize.parse(string.toUpperCase()); } @Override public PrintWriter getOut() { return out; } @Override public void setOut(PrintWriter out) { this.out = out; } @Override public PrintWriter getErr() { return err; } @Override public void setErr(PrintWriter err) { this.err = err; } } ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/NoopItemWriter.java ================================================ package com.redis.riot.core; import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ItemWriter; public class NoopItemWriter implements ItemWriter { @Override public void write(Chunk items) { // Do nothing } } ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/PrintExceptionMessageHandler.java ================================================ package com.redis.riot.core; import picocli.CommandLine; import picocli.CommandLine.IExecutionExceptionHandler; import picocli.CommandLine.ParseResult; public class PrintExceptionMessageHandler implements IExecutionExceptionHandler { public int handleExecutionException(Exception exception, CommandLine cmd, ParseResult parseResult) { Throwable finalException = unwrapException(exception); if (cmd.getCommand() instanceof BaseCommand) { if (((BaseCommand) cmd.getCommand()).loggingMixin.isStacktrace()) { finalException.printStackTrace(cmd.getErr()); } } // bold red error message cmd.getErr().println(cmd.getColorScheme().errorText(finalException.getMessage())); return cmd.getExitCodeExceptionMapper() != null ? cmd.getExitCodeExceptionMapper().getExitCode(finalException) : cmd.getCommandSpec().exitCodeOnExecutionException(); } private Throwable unwrapException(Exception exception) { if (exception instanceof RiotException) { RiotException riotException = (RiotException) exception; if (riotException.getCause() == null) { return riotException; } return riotException.getCause(); } return exception; } } ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/ProcessingItemWriter.java ================================================ package com.redis.riot.core; import org.springframework.batch.item.Chunk; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemStream; import org.springframework.batch.item.ItemStreamException; import org.springframework.batch.item.ItemStreamSupport; import org.springframework.batch.item.ItemStreamWriter; import org.springframework.batch.item.ItemWriter; import org.springframework.util.ClassUtils; public class ProcessingItemWriter extends ItemStreamSupport implements ItemStreamWriter { private final ItemProcessor processor; private final ItemWriter writer; public ProcessingItemWriter(ItemProcessor processor, ItemWriter writer) { setName(ClassUtils.getShortName(getClass())); this.processor = processor; this.writer = writer; } @Override public void open(ExecutionContext executionContext) throws ItemStreamException { if (processor instanceof ItemStream) { ((ItemStream) processor).open(executionContext); } if (writer instanceof ItemStream) { ((ItemStream) writer).open(executionContext); } } @Override public void close() throws ItemStreamException { if (writer instanceof ItemStream) { ((ItemStream) writer).close(); } if (processor instanceof ItemStream) { ((ItemStream) processor).close(); } } @Override public void update(ExecutionContext executionContext) throws ItemStreamException { if (processor instanceof ItemStream) { ((ItemStream) processor).update(executionContext); } if (writer instanceof ItemStream) { ((ItemStream) writer).update(executionContext); } } @Override public void write(Chunk chunk) throws Exception { Chunk processedChunk = new Chunk<>(); for (S item : chunk) { T processedItem = processor.process(item); if (processedItem != null) { processedChunk.add(processedItem); } } writer.write(processedChunk); } } ================================================ FILE: core/riot-core/src/main/java/com/redis/riot/core/ProgressArgs.java ================================================ package com.redis.riot.core; import lombok.ToString; import picocli.CommandLine.Option; @ToString public class ProgressArgs { public static final RiotDuration DEFAULT_UPDATE_INTERVAL = RiotDuration.ofSeconds(1); public static final ProgressStyle DEFAULT_STYLE = ProgressStyle.ASCII; @Option(names = "--progress", description = "Progress style: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).", paramLabel = "