Repository: gruelbox/transaction-outbox Branch: master Commit: 6e1814a539d1 Files: 198 Total size: 614.5 KB Directory structure: gitextract_2k8z5qvm/ ├── .github/ │ ├── dependabot.yml │ └── workflows/ │ ├── cd_build.yml │ ├── pull_request.yml │ └── release.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── pom.xml ├── settings.xml ├── transactionoutbox-acceptance/ │ ├── pom.xml │ └── src/ │ └── test/ │ └── java/ │ └── com/ │ └── gruelbox/ │ └── transactionoutbox/ │ └── acceptance/ │ ├── TestComplexConfigurationExample.java │ ├── TestH2.java │ ├── TestMSSqlServer2019.java │ ├── TestMSSqlServer2022.java │ ├── TestMySql5.java │ ├── TestMySql8.java │ ├── TestOracle18.java │ ├── TestOracle21.java │ ├── TestPostgres11.java │ ├── TestPostgres12.java │ ├── TestPostgres13.java │ ├── TestPostgres14.java │ ├── TestPostgres15.java │ ├── TestPostgres16.java │ ├── TestRequestSerialization.java │ ├── TestStubbing.java │ └── persistor/ │ ├── TestDefaultPersistorH2.java │ ├── TestDefaultPersistorMSSqlServer2019.java │ ├── TestDefaultPersistorMySql5.java │ ├── TestDefaultPersistorMySql8.java │ ├── TestDefaultPersistorOracle18.java │ └── TestDefaultPersistorPostgres16.java ├── transactionoutbox-core/ │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ ├── AlreadyScheduledException.java │ │ ├── ConnectionProvider.java │ │ ├── DataSourceConnectionProvider.java │ │ ├── DefaultDialect.java │ │ ├── DefaultInvocationSerializer.java │ │ ├── DefaultMigrationManager.java │ │ ├── DefaultPersistor.java │ │ ├── Dialect.java │ │ ├── DriverConnectionProvider.java │ │ ├── ExecutorSubmitter.java │ │ ├── FailedDeserializingInvocation.java │ │ ├── FunctionInstantiator.java │ │ ├── Instantiator.java │ │ ├── Invocation.java │ │ ├── InvocationSerializer.java │ │ ├── Migration.java │ │ ├── MissingOptionalDependencyException.java │ │ ├── NoTransactionActiveException.java │ │ ├── OptimisticLockException.java │ │ ├── ParameterContextTransactionManager.java │ │ ├── Persistor.java │ │ ├── ReflectionInstantiator.java │ │ ├── RuntimeTypeAdapterFactory.java │ │ ├── SQLAction.java │ │ ├── SimpleTransactionManager.java │ │ ├── StubParameterContextTransactionManager.java │ │ ├── StubPersistor.java │ │ ├── StubThreadLocalTransactionManager.java │ │ ├── Submitter.java │ │ ├── ThreadLocalContextTransactionManager.java │ │ ├── ThrowingRunnable.java │ │ ├── ThrowingTransactionalSupplier.java │ │ ├── ThrowingTransactionalWork.java │ │ ├── Transaction.java │ │ ├── TransactionContextPlaceholder.java │ │ ├── TransactionManager.java │ │ ├── TransactionOutbox.java │ │ ├── TransactionOutboxEntry.java │ │ ├── TransactionOutboxImpl.java │ │ ├── TransactionOutboxListener.java │ │ ├── TransactionalInvocation.java │ │ ├── TransactionalSupplier.java │ │ ├── TransactionalWork.java │ │ ├── UncheckedException.java │ │ ├── Validatable.java │ │ ├── Validator.java │ │ └── spi/ │ │ ├── AbstractFullyQualifiedNameInstantiator.java │ │ ├── AbstractThreadLocalTransactionManager.java │ │ ├── ProxyFactory.java │ │ ├── SimpleTransaction.java │ │ └── Utils.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ ├── AbstractTestDefaultInvocationSerializer.java │ │ ├── TestDefaultInvocationSerializer.java │ │ ├── TestDefaultMigrationManager.java │ │ ├── TestDefaultPersistorConfiguration.java │ │ ├── TestProxyGeneration.java │ │ └── TestValidator.java │ └── resources/ │ └── logback-test.xml ├── transactionoutbox-guice/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ └── guice/ │ │ └── GuiceInstantiator.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ └── guice/ │ │ └── acceptance/ │ │ ├── TestGuiceBinding.java │ │ └── TestGuiceInstantiator.java │ └── resources/ │ └── logback-test.xml ├── transactionoutbox-jackson/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ └── jackson/ │ │ ├── CustomInvocationDeserializer.java │ │ ├── CustomInvocationSerializer.java │ │ ├── JacksonInvocationSerializer.java │ │ ├── TransactionOutboxEntryDeserializer.java │ │ └── TransactionOutboxJacksonModule.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ └── jackson/ │ │ ├── MonetaryAmount.java │ │ ├── SerializationStressTestInput.java │ │ ├── TestJacksonInvocationSerializer.java │ │ ├── TestTransactionOutboxEntrySerialization.java │ │ └── acceptance/ │ │ └── TestJacksonSerializer.java │ └── resources/ │ └── logback-test.xml ├── transactionoutbox-jooq/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ └── jooq/ │ │ ├── DefaultJooqTransactionManager.java │ │ ├── JooqTransactionListener.java │ │ ├── JooqTransactionManager.java │ │ └── ThreadLocalJooqTransactionManager.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ └── jooq/ │ │ └── acceptance/ │ │ ├── AbstractJooqAcceptanceTest.java │ │ ├── AbstractJooqAcceptanceThreadLocalTest.java │ │ ├── JooqTestUtils.java │ │ ├── TestJooqThreadLocalH2.java │ │ ├── TestJooqThreadLocalMSSqlServer2019.java │ │ ├── TestJooqThreadLocalMySql5.java │ │ ├── TestJooqThreadLocalMySql8.java │ │ ├── TestJooqThreadLocalPostgres16.java │ │ ├── TestJooqTransactionManagerWithDefaultProviderAndExplicitlyPassedContext.java │ │ └── TestJooqTransactionManagerWithDefaultProviderAndThreadLocalContext.java │ └── resources/ │ └── logback-test.xml ├── transactionoutbox-quarkus/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ └── quarkus/ │ │ ├── CdiInstantiator.java │ │ └── QuarkusTransactionManager.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ └── quarkus/ │ │ └── acceptance/ │ │ ├── ApplicationConfig.java │ │ ├── BusinessService.java │ │ ├── BusinessServiceTest.java │ │ ├── DaoImpl.java │ │ └── RemoteCallService.java │ └── resources/ │ ├── application.properties │ └── db/ │ └── create.sql ├── transactionoutbox-spring/ │ ├── README.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ └── spring/ │ │ ├── SpringInstantiator.java │ │ ├── SpringTransactionManager.java │ │ └── SpringTransactionOutboxConfiguration.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ └── spring/ │ │ ├── SpringTransactionManagerTest.java │ │ ├── example/ │ │ │ ├── multipledatasources/ │ │ │ │ ├── EventuallyConsistentController.java │ │ │ │ ├── ExternalsConfiguration.java │ │ │ │ ├── MultipleDataSourcesTest.java │ │ │ │ ├── TransactionOutboxBackgroundProcessor.java │ │ │ │ ├── TransactionOutboxProperties.java │ │ │ │ ├── TransactionOutboxSpringMultipleDatasourcesDemoApplication.java │ │ │ │ ├── computer/ │ │ │ │ │ ├── Computer.java │ │ │ │ │ ├── ComputerExternalQueueService.java │ │ │ │ │ ├── ComputerRepository.java │ │ │ │ │ └── ComputersDbConfiguration.java │ │ │ │ └── employee/ │ │ │ │ ├── Employee.java │ │ │ │ ├── EmployeeExternalQueueService.java │ │ │ │ ├── EmployeeRepository.java │ │ │ │ └── EmployeesDbConfiguration.java │ │ │ └── simple/ │ │ │ ├── Customer.java │ │ │ ├── CustomerRepository.java │ │ │ ├── EventuallyConsistentController.java │ │ │ ├── EventuallyConsistentControllerTest.java │ │ │ ├── ExternalQueueService.java │ │ │ ├── ExternalsConfiguration.java │ │ │ ├── TransactionOutboxBackgroundProcessor.java │ │ │ ├── TransactionOutboxProperties.java │ │ │ ├── TransactionOutboxSpringDemoApplication.java │ │ │ └── Utils.java │ │ └── it/ │ │ ├── MyRemoteService.java │ │ ├── SpringTransactionManagerIT.java │ │ └── TestApplication.java │ └── resources/ │ ├── META-INF/ │ │ └── persistence.xml │ ├── application.properties │ └── logback-test.xml ├── transactionoutbox-testing/ │ ├── pom.xml │ └── src/ │ └── main/ │ ├── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ ├── TestingMode.java │ │ └── testing/ │ │ ├── AbstractAcceptanceTest.java │ │ ├── AbstractPersistorTest.java │ │ ├── BaseTest.java │ │ ├── ClassProcessor.java │ │ ├── InterfaceProcessor.java │ │ ├── LatchListener.java │ │ ├── OrderedEntryListener.java │ │ ├── ProcessedEntryListener.java │ │ └── TestUtils.java │ └── resources/ │ └── logback-test.xml ├── transactionoutbox-virtthreads/ │ ├── pom.xml │ └── src/ │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── gruelbox/ │ │ └── transactionoutbox/ │ │ └── virtthreads/ │ │ ├── AbstractVirtualThreadsTest.java │ │ ├── TestVirtualThreadsH2.java │ │ ├── TestVirtualThreadsH2Jooq.java │ │ ├── TestVirtualThreadsMySql5.java │ │ ├── TestVirtualThreadsMySql8.java │ │ ├── TestVirtualThreadsOracle21.java │ │ └── TestVirtualThreadsPostgres16.java │ └── resources/ │ └── logback-test.xml └── ~/ ├── settings.xml └── toolchains.xml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: maven directory: "/" schedule: interval: daily open-pull-requests-limit: 10 ================================================ FILE: .github/workflows/cd_build.yml ================================================ name: Continous Delivery on: push: branches: [master] jobs: build: if: "!contains(github.event.head_commit.message, 'skip ci')" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up JDK 25 uses: actions/setup-java@v4 with: distribution: 'zulu' java-package: jdk java-version: 25 server-id: github # Value of the distributionManagement/repository/id field of the pom.xml settings-path: ${{ github.workspace }} # location for the settings.xml file cache: 'maven' - name: Build, publish to GPR and tag run: | if [ "$GITHUB_REPOSITORY" == "gruelbox/transaction-outbox" ]; then revision="7.0.$GITHUB_RUN_NUMBER" echo "Building $revision at $GITHUB_SHA" mvn -Pdelombok,only-nodb-tests -B deploy -s $GITHUB_WORKSPACE/settings.xml -Drevision="$revision" -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn echo "Tagging $revision" git tag $revision git push origin $revision else mvn -Pdelombok,only-nodb-tests -B package -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn fi env: GITHUB_TOKEN: ${{ github.token }} ================================================ FILE: .github/workflows/pull_request.yml ================================================ name: Pull request on: pull_request: branches: [ master ] jobs: build: if: "!contains(github.event.head_commit.message, 'skip ci')" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: distribution: 'zulu' java-package: jdk java-version: 25 cache: maven - name: Build run: mvn -Pdelombok -B fmt:check package test-compile -DskipTests -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - uses: actions/upload-artifact@v4 with: name: build path: "**/*" compression-level: 9 test: needs: build if: "!contains(github.event.head_commit.message, 'skip ci')" runs-on: ubuntu-latest strategy: matrix: jdk: [ 17,21,25 ] db: [ nodb,mysql5,mysql8,postgres,oracle18,oracle21,mssqlserver ] fail-fast: false steps: - uses: actions/download-artifact@v4 with: name: build path: . - uses: actions/setup-java@v4 with: distribution: 'zulu' java-package: jdk java-version: ${{ matrix.jdk }} cache: maven - name: test run: mvn -Pnoformat,only-${{ matrix.db }}-tests -B test -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn ================================================ FILE: .github/workflows/release.yml ================================================ name: Publish to Central on: release: types: [created] jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up JDK 25 uses: actions/setup-java@v4 with: distribution: 'zulu' java-package: jdk java-version: 25 settings-path: "~" # location for the settings.xml file cache: 'maven' - name: Build and publish run: | set -e revision=${GITHUB_REF##*/} echo "Publishing version $revision to Central" echo ${{ secrets.GPG_SECRETKEYS }} | base64 --decode | $GPG_EXECUTABLE --import --no-tty --batch --yes echo ${{ secrets.GPG_OWNERTRUST }} | base64 --decode | $GPG_EXECUTABLE --import-ownertrust --no-tty --batch --yes sed -i "s_\(\)[^<]*_\1${revision}_g" pom.xml mvn -Prelease,delombok -B deploy -s $GITHUB_WORKSPACE/settings.xml -Drevision=$revision -DskipTests -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn env: GITHUB_TOKEN: ${{ github.token }} SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} GPG_EXECUTABLE: gpg GPG_KEYNAME: ${{ secrets.GPG_KEYNAME }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - name: Update READMEs run: | set -e revision=${GITHUB_REF##*/} echo "Updating READMEs" sed -i "s_\(\)[^<]*_\1${revision}_g" README.md sed -i "s_\(\)[^<]*_\1${revision}_g" transactionoutbox-guice/README.md sed -i "s_\(\)[^<]*_\1${revision}_g" transactionoutbox-jackson/README.md sed -i "s_\(\)[^<]*_\1${revision}_g" transactionoutbox-jooq/README.md sed -i "s_\(\)[^<]*_\1${revision}_g" transactionoutbox-spring/README.md sed -i "s_\(\)[^<]*_\1${revision}_g" transactionoutbox-quarkus/README.md sed -i "s_\(implementation 'com.gruelbox:transactionoutbox-core:\)[^']*_\1${revision}_g" README.md sed -i "s_\(implementation 'com.gruelbox:transactionoutbox-guice:\)[^']*_\1${revision}_g" transactionoutbox-guice/README.md sed -i "s_\(implementation 'com.gruelbox:transactionoutbox-jackson:\)[^']*_\1${revision}_g" transactionoutbox-jackson/README.md sed -i "s_\(implementation 'com.gruelbox:transactionoutbox-jooq:\)[^']*_\1${revision}_g" transactionoutbox-jooq/README.md sed -i "s_\(implementation 'com.gruelbox:transactionoutbox-spring:\)[^']*_\1${revision}_g" transactionoutbox-spring/README.md sed -i "s_\(implementation 'com.gruelbox:transactionoutbox-quarkus:\)[^']*_\1${revision}_g" transactionoutbox-quarkus/README.md - name: Create version update pull request uses: gruelbox/create-pull-request@master with: commit-message: "Update versions in READMEs [skip ci]" title: Update versions in READMEs body: Updates the versions in the README files following the release branch: update-readme-version base: master author: GitHub ================================================ FILE: .gitignore ================================================ **/.classpath **/.project **/.settings **/.factorypath /.idea /*.iml **/target/** *.iml **/.flattened-pom.xml ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at . All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # transaction-outbox [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.gruelbox/transactionoutbox-core/badge.svg)](#stable-releases) [![Javadocs](https://www.javadoc.io/badge/com.gruelbox/transactionoutbox-core.svg?color=blue)](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-core) [![GitHub Release Date](https://img.shields.io/github/release-date/gruelbox/transaction-outbox)](https://github.com/gruelbox/transaction-outbox/releases/latest) [![Latest snapshot](https://img.shields.io/github/v/tag/gruelbox/transaction-outbox?label=snapshot&sort=semver)](#development-snapshots) [![GitHub last commit](https://img.shields.io/github/last-commit/gruelbox/transaction-outbox)](https://github.com/gruelbox/transaction-outbox/commits/master) [![CD](https://github.com/gruelbox/transaction-outbox/workflows/Continous%20Delivery/badge.svg)](https://github.com/gruelbox/transaction-outbox/actions) [![CodeFactor](https://www.codefactor.io/repository/github/gruelbox/transaction-outbox/badge)](https://www.codefactor.io/repository/github/gruelbox/transaction-outbox) A flexible implementation of the [Transaction Outbox Pattern](https://microservices.io/patterns/data/transactional-outbox.html) for Java. `TransactionOutbox` has a clean, extensible API, very few dependencies and plays nicely with a variety of database platforms, transaction management approaches and application frameworks. Every aspect is highly configurable or overridable. It features out-of-the-box support for **Spring DI**, **Spring Txn**, **Guice**, **MySQL 5 & 8**, **PostgreSQL 11-16**, **Oracle 18 & 21**, **MS SQL Server 2017** and **H2**. ## Contents 1. [Why do I need it?](#why-do-i-need-it) 1. [Installation](#installation) 1. [Requirements](#requirements) 1. [Stable releases](#stable-releases) 1. [Development snapshots](#development-snapshots) 1. [Basic Configuration](#basic-configuration) 1. [No existing transaction manager or dependency injection](#no-existing-transaction-manager-or-dependency-injection) 1. [Spring](#spring) 1. [Guice](#guice) 1. [jOOQ](#jooq) 1. [Set up the background worker](#set-up-the-background-worker) 1. [Managing the "dead letter queue"](#managing-the-dead-letter-queue) 1. [Advanced](#advanced) 1. [Topics and FIFO ordering](#topics-and-fifo-ordering) 1. [The nested outbox pattern](#the-nested-outbox-pattern) 1. [Idempotency protection](#idempotency-protection) 1. [Delayed/scheduled processing](#delayedscheduled-processing) 1. [Flexible serialization](#flexible-serialization-beta) 1. [Clustering](#clustering) 1. [OpenTelemetry](#opentelemetry) 1. [Encryption](#encryption) 1. [Configuration reference](#configuration-reference) 1. [Stubbing in tests](#stubbing-in-tests) ## Why do I need it? [This article](https://microservices.io/patterns/data/transactional-outbox.html) explains the concept in an abstract manner, but let's say we have a microservice that handles point-of-sale and need to implement a REST endpoint to record a sale. We end up with this: ### Attempt 1 ```java @POST @Path("/sales") @Transactional public SaleId createWidget(Sale sale) { var saleId = saleRepository.save(sale); messageQueue.postMessage(StockReductionEvent.of(sale.item(), sale.amount())); messageQueue.postMessage(IncomeEvent.of(sale.value())); return saleId; } ``` The `SaleRepository` handles recording the sale in the customer's account, the `StockReductionEvent` goes off to our _warehouse_ service, and the `IncomeEvent` goes to our financial records service (let's ignore the potential flaws in the domain modelling for now). There's a big problem here: the `@Transactional` annotation is a lie (no, [really](https://lmgtfy.com/?q=dont+use+distributed+transactions)). It only really wraps the `SaleRepository` call, but not the two event postings. This means that we could end up sending the two events and fail to actually commit the sale. Our system is now inconsistent. ### Attempt 2 - Use Idempotency We could make whole method [idempotent](http://restcookbook.com/HTTP%20Methods/idempotency/) and re-write it to work a bit more like this: ```java @PUT @Path("/sales/{id}") public void createWidget(@PathParam("id") SaleId saleId, Sale sale) { saleRepository.saveInNewTransaction(saleId, sale); messageQueue.postMessage(StockReductionEvent.of(saleId, sale.item(), sale.amount())); messageQueue.postMessage(IncomeEvent.of(saleId, sale.value())); } ``` This is better. As long as the caller keeps calling the method until they get a success, we can keep re-saving and re-sending the messages without any risk of duplicating work. This works regardless of the order of the calls (and in any case, there may be good reasons of referential integrity to fix the order). The problem is that _they might stop trying_, and if they do, we could end up with only part of this transaction completed. If this is a public API, we can't force clients to use it correctly. We also still have another problem: external calls are inherently more vulnerable to downtime and performance degredation. We could find our service rendered unresponsive or failing if they are unavailable. Ideally, we would like to "buffer" these external calls within our service safely until our downstream dependencies are available. ### Attempt 3 - Transaction Outbox Idempotency is a good thing, so let's stick with the `PUT`. Here is the same example, using Transaction Outbox: ```java @PUT @Path("/sales/{id}") @Transactional public void createWidget(@PathParam("id") SaleId saleId, Sale sale) { saleRepository.save(saleId, sale); MessageQueue proxy = transactionOutbox.schedule(MessageQueue.class); proxy.postMessage(StockReductionEvent.of(saleId, sale.item(), sale.amount())); proxy.postMessage(IncomeEvent.of(saleId, sale.value())); } ``` Here's what happens: - When you create an instance of [`TransactionOutbox`](https://www.javadoc.io/static/com.gruelbox/transactionoutbox-core/0.1.57/com/gruelbox/transactionoutbox/TransactionOutbox.html) (see [Basic Configuration](#basic-configuration)), it will, by default, automatically create two database tables, `TXNO_OUTBOX` and `TXNO_VERSION`, and then keep these synchronized with schema changes as new versions are released. _Note: this is the default behaviour on a SQL database, but is completely overridable if you are using a different type of data store or don't want a third party library managing your database schema. See [Configuration reference](#configuration-reference)_. - [`TransactionOutbox`](https://www.javadoc.io/static/com.gruelbox/transactionoutbox-core/0.1.57/com/gruelbox/transactionoutbox/TransactionOutbox.html) creates a proxy of `MessageQueue`. Any method calls on the proxy are serialized and written to the `TXNO_OUTBOX` table (by default) _in the same transaction_ as the `SaleRepository` call. The call returns immediately rather than actually invoking the real method. - If the transaction rolls back, so do the serialized requests. - Immediately after the transaction is successfully committed, another thread will attempt to make the _real_ call to `MessageQueue` asynchronously. - If that call fails, or the application dies before the call is attempted, a [background "mop-up" thread](#set-up-the-background-worker) will re-attempt the call a configurable number of times, with configurable time between each, before [blocking](#managing-the-dead-letter-queue) the request and firing and event for it to be investigated (similar to a [dead letter queue](https://en.wikipedia.org/wiki/Dead_letter_queue)). - Blocked requests can be easily [unblocked](#managing-the-dead-letter-queue) again once the underlying issue is resolved. Our service is now resilient and explicitly eventually consistent, as long as all three elements (`SaleRepository` and the downstream event handlers) are idempotent, since those messages will be attempted repeatedly until confirmed successful, which means they could occur multiple times. If you find yourself wondering _why bother with the queues now_? You're quite right. As we now have outgoing buffers, we already have most of the benefits of middleware (at least for some use cases). We could replace the calls to a message queue with direct queues to the other services' load balancers and switch to a peer-to-peer architecture, if we so choose. > Note that for the above example to work, `StockReductionEvent` and `IncomeEvent` need to be included for serialization. See [Configuration reference](#configuration-reference). ## Installation ### Requirements - At least **Java 11**. Downgrading to requiring Java 8 is [under consideration](https://github.com/gruelbox/transaction-outbox/issues/29). - Currently, **MySQL**, **PostgreSQL**, **Oracle**, **MS SQL Server** or **H2** databases (pull requests to support other traditional RDMBS would be trivial. Beyond that, a SQL database is not strictly necessary for the pattern to work, merely a data store with the concept of a transaction spanning multiple mutation operations). - Database access via **JDBC** (In principle, JDBC should not be required - alternatives such as R2DBC are under investigation - but the API is currently tied to it) - Native transactions (not JTA or similar). - (Optional) Proxying non-interfaces requires [ByteBuddy](https://bytebuddy.net/#/) and for proxying classes without default constructors [Objenesis](http://objenesis.org/) to be added as a dependency ### Stable releases The latest stable release is available from Maven Central. Stable releases are [sort-of semantically versioned](https://semver.org/). That is, they follow semver in every respect other than that the version numbers are not monotically increasing. The project uses continuous delivery and selects individual stable releases to promote to Central, so Central releases will always be spaced apart numerically. The important thing, though, is that dependencies should be safe to upgrade as long as the major version number has not increased. #### Maven ```xml com.gruelbox transactionoutbox-core 7.0.707 ``` #### Gradle ```groovy implementation 'com.gruelbox:transactionoutbox-core:7.0.707' ``` ### Development snapshots Maven Central is updated regularly. However, if you want to stay at the bleeding edge, you can use continuously-delivered releases from [Github Package Repository](https://github.com/gruelbox/transaction-outbox/packages). These can be used from your production builds since they will never be deleted (unlike `SNAPSHOT`s). #### Maven ```xml github-transaction-outbox Gruelbox Github Repository https://maven.pkg.github.com/gruelbox/transaction-outbox ``` You will need to authenticate with Github to use Github Package Repository. Create a personal access token in [your GitHub settings](https://github.com/settings/tokens). It only needs **read:package** permissions. Then add something like the following in your Maven `settings.xml`: ```xml github-transaction-outbox ${env.GITHUB_USERNAME} ${env.GITHUB_TOKEN} ``` The above example uses environment variables, allowing you to keep the credentials out of source control, but you can hard-code them if you know what you're doing. #### Gradle ```groovy repositories { maven { name = "github-transaction-outbox" url = uri("https://maven.pkg.github.com/gruelbox/transaction-outboxY") credentials { username = $githubUserName password = $githubToken } } } ``` ## Basic Configuration An application needs a single, shared instance of [`TransactionOutbox`](https://www.javadoc.io/static/com.gruelbox/transactionoutbox-core/0.1.57/com/gruelbox/transactionoutbox/TransactionOutbox.html), which is configured using a builder on construction. This takes some time to get right, particularly if you already have a transaction management solution in your application. ### No existing transaction manager or dependency injection If you have no existing transaction management, connection pooling or dependency injection, here's a quick way to get started: ```java // Use an in-memory H2 database TransactionManager transactionManager = TransactionManager.fromConnectionDetails( "org.h2.Driver", "jdbc:h2:mem:test;MV_STORE=TRUE", "test", "test")); // Create the outbox TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(Dialect.H2)) .build(); // Start a transaction transactionManager.inTransaction(tx -> { // Save some stuff tx.connection().createStatement().execute("INSERT INTO..."); // Create an outbox request outbox.schedule(MyClass.class).myMethod("Foo", "Bar")); }); ``` Alternatively, you could create the [`TransactionManager`](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-core/latest/com/gruelbox/transactionoutbox/TransactionManager.html) from a [`DataSource`](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-core/latest/com/gruelbox/transactionoutbox/TransactionManager.html), allowing you to use a connection pooling `DataSource` such as Hikari: ```java TransactionManager transactionManager = TransactionManager.fromDataSource(dataSource); ``` In this default configuration, `MyClass` must have a default constructor so the "real" implementation can be constructed at the point the method is actually invoked (which might be on another day on another instance of the application). However, you can avoid this requirement by providing an [`Instantiator`](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-core/latest/com/gruelbox/transactionoutbox/Instantiator.html) on every instance of your application that knows how to create the objects: ```java TransactionOutbox outbox = TransactionOutbox.builder() .instantiator(Instantiator.using(clazz -> createInstanceOf(clazz))) .build(); ``` ### Spring See [transaction-outbox-spring](transactionoutbox-spring/README.md), which integrates Spring's DI and/or transaction management with `TransactionOutbox`. ### Guice See [transaction-outbox-guice](transactionoutbox-guice/README.md), which integrates Guice DI `TransactionOutbox`. ### jOOQ See [transaction-outbox-jooq](transactionoutbox-jooq/README.md), which integrates jOOQ transaction management with `TransactionOutbox`. ### Oracle Oracle database compatibility requires to configure Oracle jdbc driver using following VM argument : -Doracle.jdbc.javaNetNio=false ## Set up the background worker At the moment, if any work fails first time, it won't be retried. All we need to add is a background thread that repeatedly calls [`TransactionOutbox.flush()`](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-core/latest/com/gruelbox/transactionoutbox/TransactionOutbox.html) to pick up and reprocess stale work. How you do this is up to you; it very much depends on how background processing works in your application (a reactive solution will be very different to one based on Guava `Service`, for example). However, here is a simple example: ```java Thread backgroundThread = new Thread(() -> { while (!Thread.interrupted()) { try { // Keep flushing work until there's nothing left to flush while (outbox.flush()) {} } catch (Exception e) { log.error("Error flushing transaction outbox. Pausing", e); } try { // When we run out of work, pause for a minute before checking again Thread.sleep(60_000); } catch (InterruptedException e) { break; } } }); // Startup backgroundThread.start(); // Shut down backgroundThread.interrupt(); backgroundThread.join(); ``` `flush()` is designed to handle concurrent use on databases that support `SKIP LOCKED`, such as Postgres and MySQL 8+. Feel free to run this as often as you like (within reason, e.g. once a minute) on every instance of your application. This can have the benefit of spreading work across multiple instances when the work backlog is extremely high, but is not as effective as a proper [clustering](#clustering) approach. However, multiple concurrent calls to `flush()` can cause lock timeout errors on databases without `SKIP LOCKED` support, such as MySQL 5.7. This is harmless, but will cause a lot of log noise, so you may prefer to run on a single instance at a time to avoid this. ## Managing the "dead letter queue" Work might be retried too many times and enter a blocked state. You should set up an alert to allow you to manage this when it occurs, resolve the issue and unblock the work, since the work not being complete will usually be a sign that your system is out of sync in some way. ```java TransactionOutbox.builder() ... .listener(new TransactionOutboxListener() { @Override public void blocked(TransactionOutboxEntry entry, Throwable cause) { // Spring example applicationEventPublisher.publishEvent(new TransactionOutboxBlockedEvent(entry.getId(), cause); } }) .build(); ``` To mark the work for reprocessing, just use [`TransactionOutbox.unblock()`](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-core/latest/com/gruelbox/transactionoutbox/TransactionOutbox.html). Its failure count will be marked back down to zero and it will get reprocessed on the next call to `flush()`: ```java transactionOutboxEntry.unblock(entryId); ``` Or if using a `TransactionManager` that relies on explicit context (such as a non-thread local [`JooqTransactionManager`](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-jooq/latest/com/gruelbox/transactionoutbox/JooqTransactionManager.html)): ```java transactionOutboxEntry.unblock(entryId, context); ``` A good approach here is to use the [`TransactionOutboxListener`](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-core/latest/com/gruelbox/transactionoutbox/TransactionOutboxListener.html) callback to post an [interactive Slack message](https://api.slack.com/legacy/interactive-messages) - this can operate as both the alert and the "button" allowing a support engineer to submit the work for reprocessing. ## Advanced ### Topics and FIFO ordering For some applications, the order in which tasks are processed is important, such as when: - using the outbox to write to a FIFO queue, Kafka or AWS Kinesis topic; or - data replication, e.g. when feeding a data warehouse or distributed cache. In these scenarios, the default behaviour is unsuitable. Tasks are usually processed in a highly parallel fashion. Even if the volume of tasks is low, if a task fails and is retried later, it can easily end up processing after some later task even if that later task was processed hours or even days after the failing one. To avoid problems associated with tasks being processed out-of-order, you can order the processing of your tasks within a named "topic": ```java outbox.with().ordered("topic1").schedule(Service.class).process("red"); outbox.with().ordered("topic2").schedule(Service.class).process("green"); outbox.with().ordered("topic1").schedule(Service.class).process("blue"); outbox.with().ordered("topic2").schedule(Service.class).process("yellow"); ``` No matter what happens: - `red` will always need to be processed (successfully) before `blue`; - `green` will always need to be processed (successfully) before `yellow`; but - `red` and `blue` can run in any sequence with respect to `green` and `yellow`. This functionality was specifically designed to allow outboxed writing to Kafka topics. For maximum throughput when writing to Kafka, it is advised that you form your outbox topic name by combining the Kafka topic and partition, since that is the boundary where ordering is required. There are a number of things to consider before using this feature: - Tasks are not processed immediately when submitting, as normal, and are processed by background flushing only. This means there will be an increased delay between the source transaction being committed and the task being processed, depending on how your application calls `TransactionOutbox.flush()`. - If a task fails, no further requests will be processed _in that topic_ until a subsequent retry allows the failing task to succeed, to preserve ordered processing. This means it is possible for topics to become entirely frozen in the event that a task fails repeatedly. For this reason, it is essential to use a `TransactionOutboxListener` to watch for failing tasks and investigate quickly. Note that other topics will be unaffected. - `TransactionOutboxBuilder.blockAfterAttempts` is ignored for all tasks that use this option. - A single topic can only be processed in single-threaded fashion, but separate topics can be processed in parallel. If your tasks use a small number of topics, scalability will be affected since the degree of parallelism will be reduced. ### The nested-outbox pattern In practice, it can be extremely hard to guarantee that an entire unit of work is idempotent and thus suitable for retry. For example, the request might be to "update a customer record" with a new address, but this might record the change to an audit history table with a fresh UUID, the current date and time and so on, which in turn triggers external changes outside the transaction. The parent customer update request may be idempotent, but the downstream effects may not be. To tackle this, `TransactionOutbox` supports a use case where outbox requests spawn further outbox requests, along with a layer of additional [idempotency protection](#idempotency-protection) for particularly diffcult cases. The nested pattern works as follows: - Modify the customer record: `outbox.schedule(CustomerService.class).update(newDetails)` - The `update` method spawns a new outbox request to process the downstream effect: `outbox.schedule(AuditService.class).audit("CUSTOMER_UPDATED", UUID.randomUUID(), Instant.now(), newDetails.customerId())` Now, if any part of the top-level request throws, nothing occurs. If the top level request succeeds, an idempotent request to create the audit record will retry safely. ### Idempotency protection A common use case for `TransactionOutbox` is to receive an incoming request (such as a message from a message queue), acknowledge it immediately and process it asynchronously, for example: ```java public class FooEventHandler implements SQSEventHandler { @Inject private TransactionOutbox outbox; public void handle(ThingHappenedEvent event) { outbox.schedule(FooService.class).handleEvent(event.getThingId()); } } ``` However, incoming transports, whether they be message queues or APIs, usually need to rely on idempotency in message handlers (for the same reason that outgoing requests from outbox also rely on idempotency). This means the above code could get called twice. As long as `FooService.handleEvent()` is idempotent itself, this is harmless, but we can't always assume this. The incoming message might be a broadcast, with no knowledge of the behaviour of handlers and therefore no way of pre-generating any new record ids the handler might need and passing them in the message. To protect against this, `TransactionOutbox` can automatically detect duplicate requests and reject them with `AlreadyScheduledException`. Records of requests are retained up to a configurable threshold (see below). To use this, use the call pattern: ```java outbox.with() .uniqueRequestId("context-clientid") .schedule(Service.class) .process("Foo"); ``` Where `context-clientid` is a globally-unique identifier derived from the incoming request. Such ids are usually available from queue middleware as message ids, or if not you can require as part of the incoming API (possibly with a tenant prefix to ensure global uniqueness across tenants). ### Delayed/scheduled processing ### To delay execution of a task, use: ```java outbox.with() .delayForAtLeast(Duration.of(5, MINUTES)) .schedule(Service.class) .process("Foo"); ``` There are some caveats around how accurate timing is. See the JavaDoc on the `delayForAtLeast` method for more information. This is particularly useful when combined with the [nested outbox pattern](#the-nested-outbox-pattern) for creating polling/repeated or recursive tasks to throttle prcessing. ### Flexible serialization (beta) Most people will use the default persistor, `DefaultPersistor`, to persist tasks to a relational database. This uses `DefaultInvocationSerializer` by default, which in turn uses [GSON](https://github.com/google/gson) to serialize as JSON. `DefaultInvocationSerializer` is extremely limited by design, with a small list of allowed classes in method arguments. You can extend the list of support types by calling `serializableTypes` in its builder, but it will always be restricted to this global list. This is by design, to avoid building a [deserialization of untrusted data](https://owasp.org/www-community/vulnerabilities/Deserialization_of_untrusted_data) vulnerability into the library. Furthermore, there is no support for the case where run-time and compile-time types differ, such as in polymorphic collections. The following will always fail with `DefaultInvocationSerializer`: ```java outbox.schedule(Service.class).processList(List.of(1, "2", 3L)); ``` However, if you completely trust your serialized data (for example, your developers don't have write access to your production database, and the access credentials are well guarded) then you may prefer to have 100% flexibility, with no need to declare the types used and the ability to use any combination of run-time and compile-time types. See [transaction-outbox-jackson](transactionoutbox-jackson/README.md), which uses a specially-configured Jackson `ObjectMapper` to achieve this. ### Clustering The default mechanism for _running_ tasks (either immediately, or when they are picked up by background processing) is via a `java.concurrent.Executor`, which effectively does the following: ```java executor.execute(() -> outbox.processNow(transactionOutboxEntry)); ``` This offloads processing to a background thread _on the application instance_ on which the task was picked up. Under high load, this can mean thousands of tasks being picked up from the database queue and submitted at the same time on the same application instance, even if there are 20 instances of the application, effectively limiting the total rate of processing to what a single instance can handle. If you want to instead push the work for processing by _any_ of your application instances, thus spreading the work around a cluster, there are multiple approaches, just some of which are listed below: * An HTTP endpoint on a load-balanced DNS with service discovery (such as a container orchestrator e.g. Kubernetes or Nomad) * A shared queue (AWS SQS, ActiveMQ, ZeroMQ) * A lower-level clustering/messaging toolkit such as [JGroups](http://www.jgroups.org/). All of these can be implemented as follows: When defining the `TransactionOutbox`, replace `ExecutorSubmitter` with something which serializes a `TransactionOutboxEntry` and ships it to the remote queue/address. Here's what configuration might look for a `RestApiSubmitter` which ships the request to a load-balanced endpoint hosted on Nomad/Consul: ```java TransactionOutbox outbox = TransactionOutbox.builder().submitter(restApiSubmitter) ``` It is strongly advised that you use a local executor in-line, to ensure that if there are communications issues with your endpoint or queue, it doesn't fail the calling thread. Here is an example using [Feign](https://github.com/OpenFeign/feign): ```java @Slf4j class RestApiSubmitter implements Submitter { private final FeignResource feignResource; private final ExecutorService localExecutor; private final Provider outbox; @Inject RestApiExecutor(String endpointUrl, ExecutorService localExecutor, ObjectMapper objectMapper, Provider outbox) { this.feignResource = Feign.builder() .decoder(new JacksonDecoder(objectMapper)) .target(GitHub.class, "https://api.github.com");; this.localExecutor = localExecutor; this.outbox = outbox; } @Override public void submit(TransactionOutboxEntry entry, Consumer leIgnore) { try { localExecutor.execute(() -> processRemotely(entry)); log.info("Queued {} to be sent for remote processing", entry.description()); } catch (RejectedExecutionException e) { log.info("Will queue {} for processing when local executor is available", entry.description()); } catch (Exception e) { log.warn("Failed to queue {} for execution at {}. It will be re-attempted later.", entry.description(), url, e); } } private void processRemotely(TransactionOutboxEntry entry) { try { feignResource.process(entry); log.info("Submitted {} for remote processing at {}", entry.description(), url); } catch (Exception e) { log.warn( "Failed to submit {} for remote processing at {}. It will be re-attempted later.", entry.description(), url, e ); } } public interface FeignResource { @RequestLine("POST /outbox/process") void process(TransactionOutboxEntry entry); } } ``` Then listen on your communication mechanism for incoming serialized `TransactionOutboxEntry`s, and push them to a normal local `ExecutorSubmitter`. Here's what a JAX-RS example might look like: ```java @POST @Path("/outbox/process") void processOutboxEntry(String entry) { TransactionOutboxEntry entry = somethingWhichCanSerializeTransactionOutboxEntries.deserialize(entry); Submitter submitter = ExecutorSubmitter.builder().executor(localExecutor).logLevelWorkQueueSaturation(Level.INFO).build(); submitter.submit(entry, outbox.get()::processNow); } ``` This whole approach is complicated a little by the inherent difficulty in serializing and deserializing a `TransactionOutboxEntry`, which is extremely polymorphic in nature. A reference approach is provided by [transaction-outbox-jackson](transactionoutbox-jackson/README.md), which provides the features necessary to make a Jackson `ObjectMapper` able to handle the work. With that on the classpath you can use an `ObjectMapper` as follows: ```java // Add support for TransactionOutboxEntry to your normal application ObjectMapper yourNormalSharedObjectMapper.registerModule(new TransactionOutboxJacksonModule()); // (Optional) support deep polymorphic requests - for this we need to copy the object // mapper so it doesn't break the way the rest of your application works ObjectMapper objectMapper = yourNormalSharedObjectMapper.copy(); objectMapper.setDefaultTyping(TransactionOutboxJacksonModule.typeResolver()); // Serialize String message = objectMapper.writeValueAsString(entry); // Deserialize TransactionOutboxEntry entry = objectMapper.readValue(message, TransactionOutboxEntry.class); ``` Armed with the above, happy clustering! ### OpenTelemetry A common request is to propagate [OTEL](https://opentelemetry.io/) traces from the calling code to an outboxed task. This can be achieved using `TransactionOutboxListener` to record the details of the parent `Span` as session variables, then restore them on invocation. Example: ```java static class OtelListener implements TransactionOutboxListener { /** Serialises the current context into {@link Invocation#getSession()}. */ @Override public Map extractSession() { var result = new HashMap(); SpanContext spanContext = Span.current().getSpanContext(); if (!spanContext.isValid()) { return null; } result.put("traceId", spanContext.getTraceId()); result.put("spanId", spanContext.getSpanId()); log.info("Extracted: {}", result); return result; } /** * Deserialises {@link Invocation#getSession()} and sets it as the current context so that any * new span started by the method we invoke will treat it as the parent span */ @Override public void wrapInvocationAndInit(Invocator invocator) { Invocation inv = invocator.getInvocation(); var spanBuilder = otel.getTracer("transaction-outbox") .spanBuilder(String.format("%s.%s", inv.getClassName(), inv.getMethodName())) .setNoParent(); for (var i = 0; i < inv.getArgs().length; i++) { spanBuilder.setAttribute("arg" + i, Utils.stringify(inv.getArgs()[i])); } if (inv.getSession() != null) { var traceId = inv.getSession().get("traceId"); var spanId = inv.getSession().get("spanId"); if (traceId != null && spanId != null) { spanBuilder.addLink( SpanContext.createFromRemoteParent( traceId, spanId, TraceFlags.getSampled(), TraceState.getDefault())); } } var span = spanBuilder.startSpan(); try (Scope scope = span.makeCurrent()) { invocator.runUnchecked(); } finally { span.end(); } } } ``` Check out `AbstractAcceptanceTest.OtelListener` for an example of this in use. `extractSession` can be used for other things too, such as authentication state, HTTP session variables or anything else that might be in static context at the time of a call and you want to be inherited by the task. ### Encryption Be warned that all data written to disk is written unencrypted by default (including MDC, session variables, method arguments...). If you may be writing sensitive data, particularly authentication-related data, you are advised to create your own `InvocationSerializer` by decorating an existing implementation, and use an efficient stream encryptor to write the data in encrupted form. ## Configuration reference This example shows a number of other configuration options in action: ```java TransactionManager transactionManager = TransactionManager.fromDataSource(dataSource); TransactionOutbox outbox = TransactionOutbox.builder() // The most complex part to set up for most will be synchronizing with your existing transaction // management. Pre-rolled implementations are available for jOOQ and Spring (see above for more information) // and you can use those examples to synchronize with anything else by defining your own TransactionManager. // Or, if you have no formal transaction management at the moment, why not start, using transaction-outbox's // built-in one? .transactionManager(transactionManager) // Modify how requests are persisted to the database. For more complex modifications, you may wish to subclass // DefaultPersistor, or create a completely new Persistor implementation. .persistor(DefaultPersistor.builder() // Selecting the right SQL dialect ensures that features such as SKIP LOCKED are used correctly. .dialect(Dialect.POSTGRESQL_9) // Override the table name (defaults to "TXNO_OUTBOX") .tableName("transactionOutbox") // Shorten the time we will wait for write locks (defaults to 2) .writeLockTimeoutSeconds(1) // Disable automatic creation and migration of the outbox table, forcing the application to manage // migrations itself .migrate(false) // Allow the SaleType enum and Money class to be used in arguments (see example below) .serializer(DefaultInvocationSerializer.builder() .serializableTypes(Set.of(SaleType.class, Money.class)) .build()) .build()) // GuiceInstantiator and SpringInstantiator are great if you are using Guice or Spring DI, but what if you // have your own service locator? Wire it in here. Fully-custom Instantiator implementations are easy to // implement. .instantiator(Instantiator.using(myServiceLocator::createInstance)) // Change the log level used when work cannot be submitted to a saturated queue to INFO level (the default // is WARN, which you should probably consider a production incident). You can also change the Executor used // for submitting work to a shared thread pool used by the rest of your application. Fully-custom Submitter // implementations are also easy to implement, for example to cluster work. .submitter(ExecutorSubmitter.builder() .executor(ForkJoinPool.commonPool()) .logLevelWorkQueueSaturation(Level.INFO) .build()) // Lower the log level when a task fails temporarily from the default WARN. .logLevelTemporaryFailure(Level.INFO) // 10 attempts at a task before blocking it. .blockAfterAttempts(10) // When calling flush(), select 0.5m records at a time. .flushBatchSize(500_000) // Flush once every 15 minutes only .attemptFrequency(Duration.ofMinutes(15)) // Include Slf4j's Mapped Diagnostic Context in tasks. This means that anything in the MDC when schedule() // is called will be recreated in the task when it runs. Very useful for tracking things like user ids and // request ids across invocations. .serializeMdc(true) // Sets how long we should keep records of requests with a unique request id so duplicate requests // can be rejected. Defaults to 7 days. .retentionThreshold(Duration.ofDays(1)) // We can intercept and modify numerous events. The most common use is to catch blocked tasks // and raise alerts for these to be investigated. A Slack interactive message is particularly effective here // since it can be wired up to call unblock() automatically. .listener(new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { eventPublisher.publish(new OutboxTaskProcessedEvent(entry.getId())); } @Override public void blocked(TransactionOutboxEntry entry, Throwable cause) { eventPublisher.publish(new BlockedOutboxTaskEvent(entry.getId())); } @Override public Map extractSession() { return Map.of(); } @Override public void wrapInvocationAndInit(Invocator invocator) { invocator.runUnchecked(); } @Override public void wrapInvocation(Invocator invocator) throws Exception { invocator.run(); } }) .build(); // Usage example, using the in-built transaction manager MDC.put("SESSIONKEY", "Foo"); try { transactionManager.inTransaction(tx -> { writeSomeChanges(tx.connection()); outbox.schedule(getClass()) .performRemoteCall(SaleType.SALE, Money.of(10, Currency.getInstance("USD"))); }); } finally { MDC.clear(); } ``` ## Stubbing in tests `TransactionOutbox` should not be directly stubbed (e.g. using Mockito); the contract is too complex to stub out. Instead, [stubs](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-core/latest/com/gruelbox/transactionoutbox/StubThreadLocalTransactionManager.html) [exist](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-core/latest/com/gruelbox/transactionoutbox/StubPersistor.html) for the various arguments to the builder, allowing you to build a `TransactionOutbox` with minimal external dependencies which can be called and verified in tests. ```java // GIVEN SomeService mockService = Mockito.mock(SomeService.class); // Also see StubParameterContextTransactionManager TransactionManager transactionManager = new StubThreadLocalTransactionManager(); TransactionOutbox outbox = TransactionOutbox.builder() .instantiator(Instantiator.using(clazz -> mockService)) // Return our mock .persistor(StubPersistor.builder().build()) // Doesn't save anything .submitter(Submitter.withExecutor(MoreExecutors.directExecutor())) // Execute all work in-line .clockProvider(() -> Clock.fixed(LocalDateTime.of(2020, 3, 1, 12, 0) .toInstant(ZoneOffset.UTC), ZoneOffset.UTC)) // Fix the clock (not necessary here) .transactionManager(transactionManager) .build(); // WHEN transactionManager.inTransaction(tx -> outbox.schedule(SomeService.class).doAThing(1)); // THEN Mockito.verify(mockService).doAThing(1); ``` Depending on the type of test, you may wish to use a real `Persistor` such as `DefaultPersistor` (if there's a real database available) or a real, multi-threaded `Submitter`. That's up to you. ================================================ FILE: pom.xml ================================================ 4.0.0 com.gruelbox transactionoutbox-parent pom ${revision} Transaction Outbox Parent A safe implementation of the transactional outbox pattern for Java. 2020 https://github.com/gruelbox/transaction-outbox Graham Crockford https://gruelbox.com 17 17 UTF-8 UTF-8 UTF-8 src/main/java src/test/java false 1.5.32 1.18.42 7.0.707 6.0.3 1.21.4 3.4.0 3.2.8 3.5.4 transactionoutbox-core transactionoutbox-jackson transactionoutbox-guice transactionoutbox-testing transactionoutbox-acceptance transactionoutbox-quarkus transactionoutbox-spring org.slf4j slf4j-api 2.0.17 net.bytebuddy byte-buddy 1.18.5 true org.objenesis objenesis 3.5 true com.google.code.gson gson 2.13.2 org.projectlombok lombok ${lombok.version} provided org.junit.jupiter junit-jupiter-engine ${junit.jupiter.version} test org.junit.jupiter junit-jupiter-api ${junit.jupiter.version} test org.junit.jupiter junit-jupiter-params ${junit.jupiter.version} test org.mockito mockito-all 1.10.19 test org.hamcrest hamcrest-core 3.0 test ch.qos.logback logback-classic ${logback.version} test ch.qos.logback logback-core ${logback.version} test com.h2database h2 2.4.240 test com.zaxxer HikariCP 7.0.2 test org.testcontainers testcontainers ${testcontainers.version} test org.testcontainers junit-jupiter ${testcontainers.version} test org.testcontainers postgresql ${testcontainers.version} test org.testcontainers mysql ${testcontainers.version} test org.postgresql postgresql 42.7.10 test com.mysql mysql-connector-j 9.6.0 test org.testcontainers oracle-xe ${testcontainers.version} test com.oracle.database.jdbc ojdbc11 23.26.1.0.0 test org.testcontainers mssqlserver ${testcontainers.version} test com.microsoft.sqlserver mssql-jdbc 13.3.1.jre11-preview test io.opentelemetry opentelemetry-sdk-testing 1.58.0 test maven-compiler-plugin 3.15.0 ${maven.compiler.source} ${maven.compiler.target} true org.projectlombok lombok ${lombok.version} -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED org.apache.maven.plugins maven-jar-plugin 3.5.0 maven-surefire-plugin ${surefire.version} -Doracle.jdbc.javaNetNio=false -XX:+EnableDynamicAgentLoading org.codehaus.mojo flatten-maven-plugin 1.7.3 oss flatten process-resources flatten flatten.clean clean clean com.spotify.fmt fmt-maven-plugin 2.29 true .*\.java false false validate format au.com.acegi xml-format-maven-plugin 4.1.0 java-21-modules [21,) transactionoutbox-jooq java-25-modules [25,) transactionoutbox-virtthreads release gpg2 org.apache.maven.plugins maven-source-plugin ${maven.source.plugin.version} attach-sources jar org.sonatype.central central-publishing-maven-plugin 0.10.0 true central true published org.apache.maven.plugins maven-gpg-plugin ${maven.gpg.plugin.version} sign-artifacts verify sign ${gpg.keyname} ${gpg.keyname} --pinentry-mode loopback ossrh https://oss.sonatype.org/content/repositories/snapshots ossrh https://oss.sonatype.org/service/local/staging/deploy/maven2/ delombok target/generated-sources/delombok true projectlombok.org https://projectlombok.org/edge-releases org.projectlombok lombok-maven-plugin 1.18.20.0 org.projectlombok lombok ${lombok.version} delombok generate-sources delombok false src/main/java org.apache.maven.plugins maven-javadoc-plugin 3.12.0 true true ${project.version} target/generated-sources/delombok attach-javadocs jar only-nodb-tests maven-surefire-plugin **/*Oracle*.java **/*Postgres*.java **/*MySql5*.java **/*MySql8*.java **/*MSSqlServer*.java only-oracle18-tests maven-surefire-plugin **/*Oracle18*.java only-oracle21-tests maven-surefire-plugin **/*Oracle21*.java only-postgres-tests maven-surefire-plugin **/*Postgres*.java only-mysql5-tests maven-surefire-plugin **/*MySql5*.java only-mysql8-tests maven-surefire-plugin **/*MySql8*.java only-mssqlserver-tests maven-surefire-plugin **/*MSSqlServer*.java noformat com.spotify.fmt fmt-maven-plugin none org.codehaus.mojo flatten-maven-plugin flatten none flatten.clean none The Apache License, Version 2.0 http://www.apache.org/licenses/LICENSE-2.0.txt Graham Crockford graham@gruelbox.com Gruelbox https://gruelbox.com GitHub Issues https://github.com/gruelbox/transaction-outbox/issues scm:git:https://github.com/gruelbox/transaction-outbox.git scm:git:git@github.com:gruelbox/transaction-outbox.git https://github.com/gruelbox/transaction-outbox HEAD github GitHub OWNER Apache Maven Packages https://maven.pkg.github.com/gruelbox/transaction-outbox ================================================ FILE: settings.xml ================================================ github ${env.GITHUB_ACTOR} ${env.GITHUB_TOKEN} central ${env.SONATYPE_USERNAME} ${env.SONATYPE_PASSWORD} ossrh true ${env.GPG_KEYNAME} ${env.GPG_EXECUTABLE} ${env.GPG_PASSPHRASE} ================================================ FILE: transactionoutbox-acceptance/pom.xml ================================================ transactionoutbox-parent com.gruelbox ${revision} 4.0.0 Transaction Outbox Acceptance Tests jar transactionoutbox-acceptance A safe implementation of the transactional outbox pattern for Java (core library) com.gruelbox transactionoutbox-core ${project.version} test org.projectlombok lombok com.gruelbox transactionoutbox-testing ${project.version} test org.testcontainers testcontainers org.testcontainers junit-jupiter org.testcontainers postgresql org.testcontainers oracle-xe org.testcontainers mysql org.postgresql postgresql com.oracle.database.jdbc ojdbc11 com.mysql mysql-connector-j com.h2database h2 org.testcontainers mssqlserver com.microsoft.sqlserver mssql-jdbc ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestComplexConfigurationExample.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.*; import java.sql.Connection; import java.time.Duration; import java.util.Currency; import java.util.Set; import java.util.concurrent.ForkJoinPool; import javax.sql.DataSource; import lombok.Value; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.MDC; import org.slf4j.event.Level; /** * Just syntax-checks the example given in the README to give a warning if the example needs to * change. */ class TestComplexConfigurationExample { @Test @Disabled void test() { DataSource dataSource = Mockito.mock(DataSource.class); ServiceLocator myServiceLocator = Mockito.mock(ServiceLocator.class); EventPublisher eventPublisher = Mockito.mock(EventPublisher.class); TransactionManager transactionManager = TransactionManager.fromDataSource(dataSource); TransactionOutbox outbox = TransactionOutbox.builder() // The most complex part to set up for most will be synchronizing with your existing // transaction // management. Pre-rolled implementations are available for jOOQ and Spring (see above // for more information) // and you can use those examples to synchronize with anything else by defining your own // TransactionManager. // Or, if you have no formal transaction management at the moment, why not start, using // transaction-outbox's // built-in one? .transactionManager(transactionManager) // Modify how requests are persisted to the database. .persistor( DefaultPersistor.builder() // Selecting the right SQL dialect ensures that features such as SKIP LOCKED are // used correctly. .dialect(Dialect.POSTGRESQL_9) // Override the table name (defaults to "TXNO_OUTBOX") .tableName("transactionOutbox") // Shorten the time we will wait for write locks (defaults to 2) .writeLockTimeoutSeconds(1) // Disable automatic creation and migration of the outbox table, forcing the // application to manage // migrations itself .migrate(false) // Allow the SaleType enum and Money class to be used in arguments (see example // below) .serializer( DefaultInvocationSerializer.builder() .serializableTypes(Set.of(SaleType.class, Money.class)) .build()) .build()) .instantiator(Instantiator.using(myServiceLocator::createInstance)) // Change the log level used when work cannot be submitted to a saturated queue to INFO // level (the default // is WARN, which you should probably consider a production incident). You can also // change the Executor used // for submitting work to a shared thread pool used by the rest of your application. // Fully-custom Submitter // implementations are also easy to implement. .submitter( ExecutorSubmitter.builder() .executor(ForkJoinPool.commonPool()) .logLevelWorkQueueSaturation(Level.INFO) .build()) // Lower the log level when a task fails temporarily from the default WARN. .logLevelTemporaryFailure(Level.INFO) // 10 attempts at a task before it is blocked (and would require intervention) .blockAfterAttempts(10) // When calling flush(), select 0.5m records at a time. .flushBatchSize(500_000) // Flush once every 15 minutes only .attemptFrequency(Duration.ofMinutes(15)) // Include Slf4j's Mapped Diagnostic Context in tasks. This means that anything in the // MDC when schedule() // is called will be recreated in the task when it runs. Very useful for tracking things // like user ids and // request ids across invocations. .serializeMdc(true) // We can intercept task successes, single failures and blocked tasks. The most common // use is // to catch blocked tasks. // and raise alerts for these to be investigated. A Slack interactive message is // particularly effective here // since it can be wired up to call unblock() automatically. .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { eventPublisher.publish(new OutboxTaskProcessedEvent(entry.getId())); } @Override public void blocked(TransactionOutboxEntry entry, Throwable cause) { eventPublisher.publish(new BlockedOutboxTaskEvent(entry.getId())); } }) .build(); // Usage example, using the in-built transaction manager MDC.put("SESSIONKEY", "Foo"); try { transactionManager.inTransaction( tx -> { writeSomeChanges(tx.connection()); outbox .schedule(getClass()) .performRemoteCall(SaleType.SALE, Money.of(10, Currency.getInstance("USD"))); }); } finally { MDC.clear(); } } void performRemoteCall( @SuppressWarnings({"unused", "SameParameterValue"}) SaleType saleType, @SuppressWarnings("unused") Money amount) {} private void writeSomeChanges(@SuppressWarnings("unused") Connection connection) {} private interface ServiceLocator { T createInstance(Class clazz); } private interface EventPublisher { void publish(Object o); } @Value private static class BlockedOutboxTaskEvent { String id; } @Value private static class OutboxTaskProcessedEvent { String id; } @SuppressWarnings("unused") private enum SaleType { SALE, REFUND } private interface Money { static Money of( @SuppressWarnings("unused") int amount, @SuppressWarnings("unused") Currency currency) { return null; } } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestH2.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.gruelbox.transactionoutbox.*; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import com.gruelbox.transactionoutbox.testing.InterfaceProcessor; import com.gruelbox.transactionoutbox.testing.LatchListener; import com.gruelbox.transactionoutbox.testing.OrderedEntryListener; import java.lang.reflect.InvocationTargetException; import java.time.Duration; import java.time.Instant; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; import org.slf4j.MDC; @SuppressWarnings("WeakerAccess") class TestH2 extends AbstractAcceptanceTest { static final ThreadLocal inWrappedInvocation = ThreadLocal.withInitial(() -> false); @Test final void delayedExecutionImmediateSubmission() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); TransactionManager transactionManager = txManager(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .instantiator(Instantiator.using(clazz -> (InterfaceProcessor) (foo, bar) -> {})) .listener(new OrderedEntryListener(latch, new CountDownLatch(1))) .persistor(Persistor.forDialect(connectionDetails().dialect())) .attemptFrequency(Duration.ofSeconds(60)) .build(); outbox.initialize(); clearOutbox(); var start = Instant.now(); transactionManager.inTransaction( () -> outbox .with() .delayForAtLeast(Duration.ofSeconds(1)) .schedule(InterfaceProcessor.class) .process(1, "bar")); assertTrue(latch.await(5, TimeUnit.SECONDS)); assertTrue(start.plus(Duration.ofSeconds(1)).isBefore(Instant.now())); } @Test final void delayedExecutionFlushOnly() throws Exception { CountDownLatch latch = new CountDownLatch(1); TransactionManager transactionManager = txManager(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .instantiator(Instantiator.using(clazz -> (InterfaceProcessor) (foo, bar) -> {})) .listener(new OrderedEntryListener(latch, new CountDownLatch(1))) .persistor(Persistor.forDialect(connectionDetails().dialect())) .attemptFrequency(Duration.ofSeconds(1)) .build(); outbox.initialize(); clearOutbox(); transactionManager.inTransaction( () -> outbox .with() .delayForAtLeast(Duration.ofSeconds(2)) .schedule(InterfaceProcessor.class) .process(1, "bar")); assertFalse(latch.await(3, TimeUnit.SECONDS)); withRunningFlusher(outbox, () -> assertTrue(latch.await(3, TimeUnit.SECONDS))); } @Test final void wrapInvocations() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); TransactionManager transactionManager = txManager(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .instantiator( Instantiator.using( clazz -> (InterfaceProcessor) (foo, bar) -> { if (!inWrappedInvocation.get()) { throw new IllegalStateException("Not in a wrapped invocation"); } })) .listener( new LatchListener(latch) .andThen( new TransactionOutboxListener() { @Override public void wrapInvocation(Invocator invocator) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { inWrappedInvocation.set(true); try { invocator.run(); } finally { inWrappedInvocation.remove(); } } })) .persistor(Persistor.forDialect(connectionDetails().dialect())) .build(); outbox.initialize(); clearOutbox(); transactionManager.inTransaction( () -> outbox.schedule(InterfaceProcessor.class).process(1, "bar")); assertTrue(latch.await(5, TimeUnit.SECONDS)); } @Test final void wrapInvocationsWithMDC() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); TransactionManager transactionManager = txManager(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .instantiator( Instantiator.using( clazz -> (InterfaceProcessor) (foo, bar) -> { if (!Boolean.parseBoolean(MDC.get("BAR"))) { throw new IllegalStateException("Not in a wrapped invocation"); } })) .listener( new LatchListener(latch) .andThen( new TransactionOutboxListener() { @Override public void wrapInvocation(Invocator invocator) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { MDC.put("BAR", "true"); try { invocator.run(); } finally { MDC.remove("BAR"); } } })) .persistor(Persistor.forDialect(connectionDetails().dialect())) .build(); outbox.initialize(); clearOutbox(); transactionManager.inTransaction( () -> outbox.schedule(InterfaceProcessor.class).process(1, "bar")); assertTrue(latch.await(5, TimeUnit.SECONDS)); } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestMSSqlServer2019.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestMSSqlServer2019 extends AbstractAcceptanceTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest") .acceptLicense() .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.MS_SQL_SERVER) .driverClassName("com.microsoft.sqlserver.jdbc.SQLServerDriver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestMSSqlServer2022.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestMSSqlServer2022 extends AbstractAcceptanceTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2022-latest") .acceptLicense() .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.MS_SQL_SERVER) .driverClassName("com.microsoft.sqlserver.jdbc.SQLServerDriver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestMySql5.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import java.util.Map; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestMySql5 extends AbstractAcceptanceTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = new MySQLContainer<>("mysql:5") .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true) .withTmpFs(Map.of("/var/lib/mysql", "rw")); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.MY_SQL_5) .driverClassName("com.mysql.cj.jdbc.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestMySql8.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import java.util.Map; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestMySql8 extends AbstractAcceptanceTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = new MySQLContainer<>("mysql:8") .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true) .withTmpFs(Map.of("/var/lib/mysql", "rw")); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.MY_SQL_8) .driverClassName("com.mysql.cj.jdbc.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestOracle18.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.OracleContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestOracle18 extends AbstractAcceptanceTest { @Container @SuppressWarnings("rawtypes") private static final JdbcDatabaseContainer container = new OracleContainer("gvenzl/oracle-xe:18-slim-faststart") .withStartupTimeout(Duration.ofHours(1)); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.ORACLE) .driverClassName("oracle.jdbc.OracleDriver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } @Override protected String createTestTable() { return "CREATE TABLE TEST_TABLE (topic VARCHAR(50), ix NUMBER, foo INTEGER, CONSTRAINT TEST_TABLE_sequencing_pk PRIMARY KEY (topic, ix))"; } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestOracle21.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.OracleContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestOracle21 extends AbstractAcceptanceTest { @Container @SuppressWarnings("rawtypes") private static final JdbcDatabaseContainer container = new OracleContainer("gvenzl/oracle-xe:21-slim-faststart") .withStartupTimeout(Duration.ofHours(1)); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.ORACLE) .driverClassName("oracle.jdbc.OracleDriver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } @Override protected String createTestTable() { return "CREATE TABLE TEST_TABLE (topic VARCHAR(50), ix NUMBER, foo INTEGER, CONSTRAINT TEST_TABLE_sequencing_pk PRIMARY KEY (topic, ix))"; } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestPostgres11.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestPostgres11 extends AbstractAcceptanceTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = (JdbcDatabaseContainer) new PostgreSQLContainer("postgres:11") .withStartupTimeout(Duration.ofHours(1)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.POSTGRESQL_9) .driverClassName("org.postgresql.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestPostgres12.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestPostgres12 extends AbstractAcceptanceTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = (JdbcDatabaseContainer) new PostgreSQLContainer("postgres:12") .withStartupTimeout(Duration.ofHours(1)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.POSTGRESQL_9) .driverClassName("org.postgresql.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestPostgres13.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestPostgres13 extends AbstractAcceptanceTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = (JdbcDatabaseContainer) new PostgreSQLContainer("postgres:13") .withStartupTimeout(Duration.ofHours(1)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.POSTGRESQL_9) .driverClassName("org.postgresql.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestPostgres14.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestPostgres14 extends AbstractAcceptanceTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = (JdbcDatabaseContainer) new PostgreSQLContainer("postgres:14") .withStartupTimeout(Duration.ofHours(1)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.POSTGRESQL_9) .driverClassName("org.postgresql.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestPostgres15.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestPostgres15 extends AbstractAcceptanceTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = (JdbcDatabaseContainer) new PostgreSQLContainer("postgres:15") .withStartupTimeout(Duration.ofHours(1)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.POSTGRESQL_9) .driverClassName("org.postgresql.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestPostgres16.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestPostgres16 extends AbstractAcceptanceTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = (JdbcDatabaseContainer) new PostgreSQLContainer("postgres:16") .withStartupTimeout(Duration.ofHours(1)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.POSTGRESQL_9) .driverClassName("org.postgresql.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestRequestSerialization.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import static org.junit.jupiter.api.Assertions.assertTrue; import com.gruelbox.transactionoutbox.*; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import com.gruelbox.transactionoutbox.testing.LatchListener; import com.gruelbox.transactionoutbox.testing.TestUtils; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.Getter; import lombok.Setter; import org.junit.jupiter.api.Test; public class TestRequestSerialization { /** * Ensures that we are serializing and deserializing any request before processing it. Otherwise * work could get processed locally successfully but fail when retried since the serialized * version of the request is not equivalent to the original. */ @Test final void workAlwaysSerialized() throws Exception { TransactionManager transactionManager = simpleTxnManager(); CountDownLatch latch = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor( DefaultPersistor.builder() .dialect(connectionDetails().dialect()) .serializer( DefaultInvocationSerializer.builder() .serializableTypes(Set.of(Arg.class)) .build()) .build()) .listener(new LatchListener(latch)) .build(); clearOutbox(); Arg arg = new Arg(); arg.hiddenData = "HIDDEN"; arg.serializedData = "SERIALIZED"; transactionManager.inTransaction(() -> outbox.schedule(ComplexProcessor.class).process(arg)); assertTrue(latch.await(15, TimeUnit.SECONDS)); } protected AbstractAcceptanceTest.ConnectionDetails connectionDetails() { return AbstractAcceptanceTest.ConnectionDetails.builder() .dialect(Dialect.H2) .driverClassName("org.h2.Driver") .url( "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DEFAULT_LOCK_TIMEOUT=60000;LOB_TIMEOUT=2000;MV_STORE=TRUE") .user("test") .password("test") .build(); } private TransactionManager simpleTxnManager() { return TransactionManager.fromConnectionDetails( connectionDetails().driverClassName(), connectionDetails().url(), connectionDetails().user(), connectionDetails().password()); } private void clearOutbox() { TestUtils.runSql(simpleTxnManager(), "DELETE FROM TXNO_OUTBOX"); } static class ComplexProcessor { public void process(Arg arg) { if (arg.hiddenData != null) { throw new IllegalStateException( "Running with state that could not possibly have been serialized"); } if (!"SERIALIZED".equals(arg.serializedData)) { throw new IllegalStateException("No serialized state"); } } } @Getter @Setter static class Arg { transient String hiddenData; String serializedData; } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/TestStubbing.java ================================================ package com.gruelbox.transactionoutbox.acceptance; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; import com.gruelbox.transactionoutbox.*; import java.math.BigDecimal; import java.time.Clock; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import lombok.Value; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; /** Checks that stubbing {@link TransactionOutbox} works cleanly. */ class TestStubbing { @Test void testStubbingWithThreadLocalContext() { StubThreadLocalTransactionManager transactionManager = new StubThreadLocalTransactionManager(); TransactionOutbox outbox = createOutbox(transactionManager); Interface.invocations.clear(); transactionManager.inTransaction( () -> { outbox .schedule(Interface.class) .doThing(1, "2", new BigDecimal[] {BigDecimal.ONE, BigDecimal.TEN}); outbox.schedule(Interface.class).doThing(2, "3", new BigDecimal[] {}); outbox.schedule(Interface.class).doThing(3, null, null); }); transactionManager.inTransaction(() -> outbox.schedule(Interface.class).doThing(4, null, null)); Object expected1 = List.of(1, "2", List.of(BigDecimal.ONE, BigDecimal.TEN)); Object expected2 = List.of(2, "3", List.of()); List expected3 = new ArrayList<>(); expected3.add(3); expected3.add(null); expected3.add(null); List expected4 = new ArrayList<>(); expected4.add(4); expected4.add(null); expected4.add(null); assertThat(Interface.invocations, contains(expected1, expected2, expected3, expected4)); } @Test void testStubbingWithExplicitContextInvalidContext() { StubParameterContextTransactionManager transactionManager = new StubParameterContextTransactionManager<>(Context.class, () -> new Context(1L)); TransactionOutbox outbox = createOutbox(transactionManager); Assertions.assertThrows( IllegalArgumentException.class, () -> transactionManager.inTransaction( tx -> outbox.schedule(Interface.class).doThing(1, new Context(2L)))); } @Test void testStubbingWithExplicitContextPassingTransaction() { StubParameterContextTransactionManager transactionManager = new StubParameterContextTransactionManager<>(Context.class, () -> new Context(1L)); TransactionOutbox outbox = createOutbox(transactionManager); Interface.invocations.clear(); transactionManager.inTransaction(tx -> outbox.schedule(Interface.class).doThing(1, tx)); assertThat(Interface.invocations, hasSize(1)); assertThat(Interface.invocations.get(0).get(0), equalTo(1)); assertThat(Interface.invocations.get(0).get(1), isA(Transaction.class)); } @Test void testStubbingWithExplicitContextPassingContext() { StubParameterContextTransactionManager transactionManager = new StubParameterContextTransactionManager<>(Context.class, () -> new Context(1L)); TransactionOutbox outbox = createOutbox(transactionManager); Interface.invocations.clear(); transactionManager.inTransaction( tx -> outbox.schedule(Interface.class).doThing(1, (Context) tx.context())); assertThat(Interface.invocations, hasSize(1)); assertThat(Interface.invocations.get(0).get(0), equalTo(1)); assertThat(Interface.invocations.get(0).get(1), isA(Context.class)); } private TransactionOutbox createOutbox(TransactionManager transactionManager) { return TransactionOutbox.builder() .instantiator(Instantiator.usingReflection()) .persistor(StubPersistor.builder().build()) .submitter(Submitter.withExecutor(Runnable::run)) .transactionManager(transactionManager) .clockProvider( () -> Clock.fixed( LocalDateTime.of(2020, 3, 1, 12, 0).toInstant(ZoneOffset.UTC), ZoneOffset.UTC)) // Fix the clock .build(); } static class Interface { static List> invocations = new ArrayList<>(); void doThing(int arg1, String arg2, BigDecimal[] arg3) { ArrayList args = new ArrayList<>(); args.add(arg1); args.add(arg2); args.add(arg3 == null ? null : Arrays.asList(arg3)); invocations.add(args); } void doThing(@SuppressWarnings("SameParameterValue") int arg1, Transaction transaction) { assertThat(transaction, notNullValue()); invocations.add(List.of(arg1, transaction)); } void doThing(@SuppressWarnings("SameParameterValue") int arg1, Context context) { assertThat(context, notNullValue()); invocations.add(List.of(arg1, context)); } } @Value static class Context { long id; } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/persistor/TestDefaultPersistorH2.java ================================================ package com.gruelbox.transactionoutbox.acceptance.persistor; import com.gruelbox.transactionoutbox.DefaultPersistor; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.TransactionManager; import com.gruelbox.transactionoutbox.testing.AbstractPersistorTest; import org.testcontainers.junit.jupiter.Testcontainers; @Testcontainers class TestDefaultPersistorH2 extends AbstractPersistorTest { private final DefaultPersistor persistor = DefaultPersistor.builder().dialect(Dialect.H2).build(); private final TransactionManager txManager = TransactionManager.fromConnectionDetails( "org.h2.Driver", "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DEFAULT_LOCK_TIMEOUT=2000;LOB_TIMEOUT=2000;MV_STORE=TRUE", "test", "test"); @Override protected DefaultPersistor persistor() { return persistor; } @Override protected TransactionManager txManager() { return txManager; } @Override protected Dialect dialect() { return Dialect.H2; } @Override public void testSkipLocked() throws Exception { // Not supported. } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/persistor/TestDefaultPersistorMSSqlServer2019.java ================================================ package com.gruelbox.transactionoutbox.acceptance.persistor; import com.gruelbox.transactionoutbox.DefaultPersistor; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.TransactionManager; import com.gruelbox.transactionoutbox.testing.AbstractPersistorTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @Testcontainers class TestDefaultPersistorMSSqlServer2019 extends AbstractPersistorTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest") .acceptLicense() .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true); private final DefaultPersistor persistor = DefaultPersistor.builder().dialect(Dialect.MS_SQL_SERVER).build(); private final TransactionManager txManager = TransactionManager.fromConnectionDetails( "com.microsoft.sqlserver.jdbc.SQLServerDriver", container.getJdbcUrl(), container.getUsername(), container.getPassword()); @Override protected DefaultPersistor persistor() { return persistor; } @Override protected TransactionManager txManager() { return txManager; } @Override protected Dialect dialect() { return Dialect.MS_SQL_SERVER; } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/persistor/TestDefaultPersistorMySql5.java ================================================ package com.gruelbox.transactionoutbox.acceptance.persistor; import com.gruelbox.transactionoutbox.DefaultPersistor; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.TransactionManager; import com.gruelbox.transactionoutbox.testing.AbstractPersistorTest; import java.time.Duration; import java.util.Map; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @Testcontainers class TestDefaultPersistorMySql5 extends AbstractPersistorTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = new MySQLContainer<>("mysql:5") .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true) .withTmpFs(Map.of("/var/lib/mysql", "rw")); private final DefaultPersistor persistor = DefaultPersistor.builder().dialect(Dialect.MY_SQL_5).build(); private final TransactionManager txManager = TransactionManager.fromConnectionDetails( "com.mysql.cj.jdbc.Driver", container.getJdbcUrl(), container.getUsername(), container.getPassword()); @Override protected DefaultPersistor persistor() { return persistor; } @Override protected TransactionManager txManager() { return txManager; } @Override protected Dialect dialect() { return Dialect.MY_SQL_5; } @Override public void testSkipLocked() throws Exception { // Not supported. } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/persistor/TestDefaultPersistorMySql8.java ================================================ package com.gruelbox.transactionoutbox.acceptance.persistor; import com.gruelbox.transactionoutbox.DefaultPersistor; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.TransactionManager; import com.gruelbox.transactionoutbox.testing.AbstractPersistorTest; import java.time.Duration; import java.util.Map; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @Testcontainers class TestDefaultPersistorMySql8 extends AbstractPersistorTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = new MySQLContainer<>("mysql:8") .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true) .withTmpFs(Map.of("/var/lib/mysql", "rw")); private final DefaultPersistor persistor = DefaultPersistor.builder().dialect(Dialect.MY_SQL_8).build(); private final TransactionManager txManager = TransactionManager.fromConnectionDetails( "com.mysql.cj.jdbc.Driver", container.getJdbcUrl(), container.getUsername(), container.getPassword()); @Override protected DefaultPersistor persistor() { return persistor; } @Override protected TransactionManager txManager() { return txManager; } @Override protected Dialect dialect() { return Dialect.MY_SQL_8; } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/persistor/TestDefaultPersistorOracle18.java ================================================ package com.gruelbox.transactionoutbox.acceptance.persistor; import com.gruelbox.transactionoutbox.DefaultPersistor; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.TransactionManager; import com.gruelbox.transactionoutbox.testing.AbstractPersistorTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.OracleContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @Testcontainers class TestDefaultPersistorOracle18 extends AbstractPersistorTest { @Container @SuppressWarnings("rawtypes") private static final JdbcDatabaseContainer container = new OracleContainer("gvenzl/oracle-xe:18-slim-faststart") .withStartupTimeout(Duration.ofHours(1)) .withReuse(true); private final DefaultPersistor persistor = DefaultPersistor.builder().dialect(Dialect.ORACLE).build(); private final TransactionManager txManager = TransactionManager.fromConnectionDetails( "oracle.jdbc.OracleDriver", container.getJdbcUrl(), container.getUsername(), container.getPassword()); @Override protected DefaultPersistor persistor() { return persistor; } @Override protected TransactionManager txManager() { return txManager; } @Override protected Dialect dialect() { return Dialect.ORACLE; } } ================================================ FILE: transactionoutbox-acceptance/src/test/java/com/gruelbox/transactionoutbox/acceptance/persistor/TestDefaultPersistorPostgres16.java ================================================ package com.gruelbox.transactionoutbox.acceptance.persistor; import com.gruelbox.transactionoutbox.DefaultPersistor; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.TransactionManager; import com.gruelbox.transactionoutbox.testing.AbstractPersistorTest; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @Testcontainers class TestDefaultPersistorPostgres16 extends AbstractPersistorTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = (JdbcDatabaseContainer) new PostgreSQLContainer("postgres:16") .withStartupTimeout(Duration.ofHours(1)) .withReuse(true); private final DefaultPersistor persistor = DefaultPersistor.builder().dialect(Dialect.POSTGRESQL_9).build(); private final TransactionManager txManager = TransactionManager.fromConnectionDetails( "org.postgresql.Driver", container.getJdbcUrl(), container.getUsername(), container.getPassword()); @Override protected DefaultPersistor persistor() { return persistor; } @Override protected TransactionManager txManager() { return txManager; } @Override protected Dialect dialect() { return Dialect.POSTGRESQL_9; } } ================================================ FILE: transactionoutbox-core/pom.xml ================================================ transactionoutbox-parent com.gruelbox ${revision} 4.0.0 Transaction Outbox Core jar transactionoutbox-core A safe implementation of the transactional outbox pattern for Java (core library) org.slf4j slf4j-api net.bytebuddy byte-buddy org.objenesis objenesis com.google.code.gson gson org.projectlombok lombok org.hamcrest hamcrest-core org.junit.jupiter junit-jupiter-engine org.junit.jupiter junit-jupiter-params org.mockito mockito-all ch.qos.logback logback-classic ch.qos.logback logback-core com.zaxxer HikariCP com.h2database h2 ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/AlreadyScheduledException.java ================================================ package com.gruelbox.transactionoutbox; import java.time.Duration; /** * Thrown when we attempt to schedule an invocation with a unique client id which has already been * used within {@link TransactionOutbox.TransactionOutboxBuilder#retentionThreshold(Duration)}. */ public class AlreadyScheduledException extends RuntimeException { AlreadyScheduledException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/ConnectionProvider.java ================================================ package com.gruelbox.transactionoutbox; import java.sql.Connection; /** * Source for JDBC connections to be provided to a {@link TransactionManager}. It is not required * for a {@link TransactionManager} to use {@link ConnectionProvider}, and when integrating with * existing applications with transaction management, it is indeed unlikely to do so. */ @SuppressWarnings("WeakerAccess") public interface ConnectionProvider { /** * Requests a new connection, or an available connection from a pool. The caller is responsible * for calling {@link Connection#close()}. * * @return The connection. */ Connection obtainConnection(); } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/DataSourceConnectionProvider.java ================================================ package com.gruelbox.transactionoutbox; import com.gruelbox.transactionoutbox.spi.Utils; import java.sql.Connection; import javax.sql.DataSource; import lombok.Builder; /** * A {@link ConnectionProvider} which requests connections from a {@link DataSource}. This is * suitable for applications using connection pools or container-provided JDBC. * *

Usage: * * 

ConnectionProvider provider = DataSourceConnectionProvider.builder()
 *   .dataSource(ds)
 *   .build()
*/ @Builder final class DataSourceConnectionProvider implements ConnectionProvider { private final DataSource dataSource; @Override public Connection obtainConnection() { return Utils.uncheckedly(dataSource::getConnection); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/DefaultDialect.java ================================================ package com.gruelbox.transactionoutbox; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; import java.util.Collection; import java.util.Map; import java.util.TreeMap; import java.util.function.Function; import java.util.stream.Stream; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; import lombok.experimental.Accessors; @AllArgsConstructor(access = AccessLevel.PRIVATE) class DefaultDialect implements Dialect { static Builder builder(String name) { return new Builder(name); } @Getter private final String name; @Getter private final String deleteExpired; @Getter private final String delete; @Getter private final String selectBatch; @Getter private final String lock; @Getter private final String checkSql; @Getter private final String fetchNextInAllTopics; @Getter private final String fetchNextInSelectedTopics; @Getter private final String fetchCurrentVersion; @Getter private final String fetchNextSequence; private final Collection migrations; @Override public String booleanValue(boolean criteriaValue) { return criteriaValue ? Boolean.TRUE.toString() : Boolean.FALSE.toString(); } @Override public void createVersionTableIfNotExists(Connection connection) throws SQLException { try (Statement s = connection.createStatement()) { s.execute( "CREATE TABLE IF NOT EXISTS TXNO_VERSION (id INT DEFAULT 0, version INT, PRIMARY KEY (id))"); } } @Override public String toString() { return name; } @Override public Stream getMigrations() { return migrations.stream(); } @Setter @Accessors(fluent = true) static final class Builder { private final String name; private String delete = "DELETE FROM {{table}} WHERE id = ? and version = ?"; private String deleteExpired = "DELETE FROM {{table}} WHERE nextAttemptTime < ? AND processed = true AND blocked = false" + " LIMIT {{batchSize}}"; private String selectBatch = "SELECT {{allFields}} FROM {{table}} WHERE nextAttemptTime < ? " + "AND blocked = false AND processed = false AND topic = '*' LIMIT {{batchSize}}"; private String lock = "SELECT id, invocation FROM {{table}} WHERE id = ? AND version = ? FOR UPDATE"; private String checkSql = "SELECT 1"; private Map migrations; private Function booleanValueFrom; private SQLAction createVersionTableBy; private String fetchNextInAllTopics = "SELECT {{allFields}} FROM {{table}} a" + " WHERE processed = false AND topic <> '*' AND nextAttemptTime < ?" + " AND seq = (" + "SELECT MIN(seq) FROM {{table}} b WHERE b.topic=a.topic AND b.processed = false" + ") LIMIT {{batchSize}}"; private String fetchNextInSelectedTopics = "SELECT {{allFields}} FROM {{table}} a" + " WHERE processed = false AND topic IN ({{topicNames}}) AND nextAttemptTime < ?" + " AND seq = (" + "SELECT MIN(seq) FROM {{table}} b WHERE b.topic=a.topic AND b.processed = false" + ") LIMIT {{batchSize}}"; private String fetchCurrentVersion = "SELECT version FROM TXNO_VERSION FOR UPDATE"; private String fetchNextSequence = "SELECT seq FROM TXNO_SEQUENCE WHERE topic = ? FOR UPDATE"; Builder(String name) { this.name = name; this.migrations = new TreeMap<>(); migrations.put( 1, new Migration( 1, "Create outbox table", "CREATE TABLE TXNO_OUTBOX (\n" + " id VARCHAR(36) PRIMARY KEY,\n" + " invocation TEXT,\n" + " nextAttemptTime TIMESTAMP(6),\n" + " attempts INT,\n" + " blacklisted BOOLEAN,\n" + " version INT\n" + ")")); migrations.put( 2, new Migration( 2, "Add unique request id", "ALTER TABLE TXNO_OUTBOX ADD COLUMN uniqueRequestId VARCHAR(100) NULL UNIQUE")); migrations.put( 3, new Migration( 3, "Add processed flag", "ALTER TABLE TXNO_OUTBOX ADD COLUMN processed BOOLEAN")); migrations.put( 4, new Migration( 4, "Add flush index", "CREATE INDEX IX_TXNO_OUTBOX_1 ON TXNO_OUTBOX (processed, blacklisted, nextAttemptTime)")); migrations.put( 5, new Migration( 5, "Increase size of uniqueRequestId", "ALTER TABLE TXNO_OUTBOX MODIFY COLUMN uniqueRequestId VARCHAR(250)")); migrations.put( 6, new Migration( 6, "Rename column blacklisted to blocked", "ALTER TABLE TXNO_OUTBOX CHANGE COLUMN blacklisted blocked VARCHAR(250)")); migrations.put( 7, new Migration( 7, "Add lastAttemptTime column to outbox", "ALTER TABLE TXNO_OUTBOX ADD COLUMN lastAttemptTime TIMESTAMP(6) NULL AFTER invocation")); migrations.put( 8, new Migration( 8, "Update length of invocation column on outbox for MySQL dialects only.", "ALTER TABLE TXNO_OUTBOX MODIFY COLUMN invocation MEDIUMTEXT")); migrations.put( 9, new Migration( 9, "Add topic", "ALTER TABLE TXNO_OUTBOX ADD COLUMN topic VARCHAR(250) NOT NULL DEFAULT '*'")); migrations.put( 10, new Migration(10, "Add sequence", "ALTER TABLE TXNO_OUTBOX ADD COLUMN seq BIGINT NULL")); migrations.put( 11, new Migration( 11, "Add sequence table", "CREATE TABLE TXNO_SEQUENCE (topic VARCHAR(250) NOT NULL, seq BIGINT NOT NULL, PRIMARY KEY (topic, seq))")); migrations.put( 12, new Migration( 12, "Add flush index to support ordering", "CREATE INDEX IX_TXNO_OUTBOX_2 ON TXNO_OUTBOX (topic, processed, seq)")); migrations.put(13, new Migration(13, "Enforce UTF8 collation for outbox messages", null)); } Builder setMigration(Migration migration) { this.migrations.put(migration.getVersion(), migration); return this; } Builder changeMigration(int version, String sql) { return setMigration(this.migrations.get(version).withSql(sql)); } Builder disableMigration(@SuppressWarnings("SameParameterValue") int version) { return setMigration(this.migrations.get(version).withSql(null)); } Dialect build() { return new DefaultDialect( name, deleteExpired, delete, selectBatch, lock, checkSql, fetchNextInAllTopics, fetchNextInSelectedTopics, fetchCurrentVersion, fetchNextSequence, migrations.values()) { @Override public String booleanValue(boolean criteriaValue) { if (booleanValueFrom != null) { return booleanValueFrom.apply(criteriaValue); } return super.booleanValue(criteriaValue); } @Override public void createVersionTableIfNotExists(Connection connection) throws SQLException { if (createVersionTableBy != null) { createVersionTableBy.doAction(connection); } else { super.createVersionTableIfNotExists(connection); } } }; } } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/DefaultInvocationSerializer.java ================================================ package com.gruelbox.transactionoutbox; import com.google.gson.*; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; import com.google.gson.stream.JsonWriter; import java.io.IOException; import java.io.Reader; import java.io.Writer; import java.lang.reflect.Modifier; import java.lang.reflect.Type; import java.math.BigDecimal; import java.text.ParseException; import java.text.ParsePosition; import java.time.*; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TimeZone; import java.util.UUID; import lombok.Builder; import lombok.extern.slf4j.Slf4j; /** * A locked-down serializer which supports a limited list of primitives and simple JDK value types. * Only the following are supported: * *
    *
  • {@link Invocation} itself *
  • Primitive types such as {@code int} or {@code double} or the boxed equivalents *
  • {@link String} *
  • {@link java.util.Date} *
  • {@link java.util.UUID} *
  • The {@code java.time} classes: *
      *
    • {@link java.time.DayOfWeek} *
    • {@link java.time.Duration} *
    • {@link java.time.Instant} *
    • {@link java.time.LocalDate} *
    • {@link java.time.LocalDateTime} *
    • {@link java.time.ZonedDateTime} *
    • {@link java.time.Month} *
    • {@link java.time.MonthDay} *
    • {@link java.time.Period} *
    • {@link java.time.Year} *
    • {@link java.time.YearMonth} *
    • {@link java.time.ZoneOffset} *
    • {@link java.time.DayOfWeek} *
    • {@link java.time.temporal.ChronoUnit} *
    *
  • Arrays specifically typed to one of the above types *
  • Any types specifically passed in, which must be GSON compatible. *
*/ @Slf4j public final class DefaultInvocationSerializer implements InvocationSerializer { private final Gson gson; @Builder DefaultInvocationSerializer(Set> serializableTypes, Integer version) { this.gson = new GsonBuilder() .registerTypeAdapter( Invocation.class, new InvocationJsonSerializer( serializableTypes == null ? Set.of() : serializableTypes, version == null ? 2 : version)) .registerTypeAdapter(Date.class, new UtcDateTypeAdapter()) .registerTypeAdapter(LocalDateTime.class, new LocalDateTimeTypeAdapter()) .registerTypeAdapter(ZonedDateTime.class, new ZonedDateTimeTypeAdapter()) .registerTypeAdapter(Instant.class, new InstantTypeAdapter()) .registerTypeAdapter(Duration.class, new DurationTypeAdapter()) .registerTypeAdapter(LocalDate.class, new LocalDateTypeAdapter()) .registerTypeAdapter(MonthDay.class, new MonthDayTypeAdapter()) .registerTypeAdapter(Period.class, new PeriodTypeAdapter()) .registerTypeAdapter(Year.class, new YearTypeAdapter()) .registerTypeAdapter(YearMonth.class, new YearMonthAdapter()) .excludeFieldsWithModifiers(Modifier.TRANSIENT, Modifier.STATIC) .create(); } @Override public void serializeInvocation(Invocation invocation, Writer writer) { try { gson.toJson(invocation, writer); } catch (Exception e) { throw new IllegalArgumentException("Cannot serialize " + invocation, e); } } @Override public Invocation deserializeInvocation(Reader reader) throws IOException { try { return gson.fromJson(reader, Invocation.class); } catch (JsonIOException | JsonSyntaxException exception) { throw new IOException(exception); } } private static final class InvocationJsonSerializer implements JsonSerializer, JsonDeserializer { private final int version; private final Map, String> classToName = new HashMap<>(); private final Map> nameToClass = new HashMap<>(); InvocationJsonSerializer(Set> serializableClasses, int version) { this.version = version; addClassPair(byte.class, "byte"); addClassPair(short.class, "short"); addClassPair(int.class, "int"); addClassPair(long.class, "long"); addClassPair(float.class, "float"); addClassPair(double.class, "double"); addClassPair(boolean.class, "boolean"); addClassPair(char.class, "char"); addClass(Byte.class); addClass(Short.class); addClass(Integer.class); addClass(Long.class); addClass(Float.class); addClass(Double.class); addClass(Boolean.class); addClass(Character.class); addClass(BigDecimal.class); addClass(String.class); addClass(Date.class); addClass(UUID.class); addClass(DayOfWeek.class); addClass(Duration.class); addClass(Instant.class); addClass(LocalDate.class); addClass(LocalDateTime.class); addClass(ZonedDateTime.class); addClass(Month.class); addClass(MonthDay.class); addClass(Period.class); addClass(Year.class); addClass(YearMonth.class); addClass(ZoneOffset.class); addClass(DayOfWeek.class); addClass(ChronoUnit.class); addClass(Transaction.class); addClassPair(TransactionContextPlaceholder.class, "TransactionContext"); serializableClasses.forEach(clazz -> addClassPair(clazz, clazz.getName())); } private void addClass(Class clazz) { addClassPair(clazz, clazz.getSimpleName()); } private void addClassPair(Class clazz, String name) { classToName.put(clazz, name); nameToClass.put(name, clazz); String arrayClassName = toArrayClassName(clazz); Class arrayClass = toClass(clazz.getClassLoader(), arrayClassName); classToName.put(arrayClass, arrayClassName); nameToClass.put(arrayClassName, arrayClass); } private String toArrayClassName(Class clazz) { if (clazz.isArray()) { return "[" + clazz.getName(); } else if (clazz == boolean.class) { return "[Z"; } else if (clazz == byte.class) { return "[B"; } else if (clazz == char.class) { return "[C"; } else if (clazz == double.class) { return "[D"; } else if (clazz == float.class) { return "[F"; } else if (clazz == int.class) { return "[I"; } else if (clazz == long.class) { return "[J"; } else if (clazz == short.class) { return "[S"; } else { return "[L" + clazz.getName() + ";"; } } private Class toClass(ClassLoader classLoader, String name) { try { return classLoader != null ? Class.forName(name, false, classLoader) : Class.forName(name); } catch (ClassNotFoundException e) { throw new RuntimeException( "Cannot determine array type for " + name + " using " + (classLoader == null ? "root classloader" : "base classloader"), e); } } @Override public JsonElement serialize(Invocation src, Type typeOfSrc, JsonSerializationContext context) { if (version == 1) { log.warn("Serializing as deprecated version {}", version); return serializeV1(src, typeOfSrc, context); } JsonObject obj = new JsonObject(); obj.addProperty("c", src.getClassName()); obj.addProperty("m", src.getMethodName()); JsonArray params = new JsonArray(); JsonArray args = new JsonArray(); int i = 0; for (Class parameterType : src.getParameterTypes()) { params.add(nameForClass(parameterType)); Object arg = src.getArgs()[i]; if (arg == null) { JsonObject jsonObject = new JsonObject(); jsonObject.add("t", null); jsonObject.add("v", null); args.add(jsonObject); } else { JsonObject jsonObject = new JsonObject(); jsonObject.addProperty("t", nameForClass(arg.getClass())); jsonObject.add("v", context.serialize(arg)); args.add(jsonObject); } i++; } obj.add("p", params); obj.add("a", args); obj.add("x", context.serialize(src.getMdc())); obj.add("s", context.serialize(src.getSession())); return obj; } JsonElement serializeV1(Invocation src, Type typeOfSrc, JsonSerializationContext context) { JsonObject obj = new JsonObject(); obj.addProperty("c", src.getClassName()); obj.addProperty("m", src.getMethodName()); JsonArray params = new JsonArray(); int i = 0; for (Class parameterType : src.getParameterTypes()) { JsonObject jsonObject = new JsonObject(); jsonObject.addProperty("t", nameForClass(parameterType)); jsonObject.add("v", context.serialize(src.getArgs()[i])); params.add(jsonObject); i++; } obj.add("p", params); obj.add("x", context.serialize(src.getMdc())); return obj; } @Override public Invocation deserialize( JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { JsonObject jsonObject = json.getAsJsonObject(); String className = jsonObject.get("c").getAsString(); String methodName = jsonObject.get("m").getAsString(); JsonArray jsonParams = jsonObject.get("p").getAsJsonArray(); Class[] params = new Class[jsonParams.size()]; for (int i = 0; i < jsonParams.size(); i++) { JsonElement param = jsonParams.get(i); if (param.isJsonObject()) { // For backwards compatibility params[i] = classForName(param.getAsJsonObject().get("t").getAsString()); } else { params[i] = classForName(param.getAsString()); } } JsonElement argsElement = jsonObject.get("a"); if (argsElement == null) { // For backwards compatibility argsElement = jsonObject.get("p"); } JsonArray jsonArgs = argsElement.getAsJsonArray(); Object[] args = new Object[jsonArgs.size()]; for (int i = 0; i < jsonArgs.size(); i++) { JsonElement arg = jsonArgs.get(i); JsonElement argType = arg.getAsJsonObject().get("t"); if (argType != null) { JsonElement argValue = arg.getAsJsonObject().get("v"); Class argClass = classForName(argType.getAsString()); try { args[i] = context.deserialize(argValue, argClass); } catch (Exception e) { throw new RuntimeException( "Failed to deserialize arg [" + argValue + "] of type [" + argType + "]", e); } } } Map mdc = context.deserialize(jsonObject.get("x"), Map.class); Map session = context.deserialize(jsonObject.get("s"), Map.class); return new Invocation(className, methodName, params, args, mdc, session); } private Class classForName(String name) { var clazz = nameToClass.get(name); if (clazz == null) { throw new IllegalArgumentException("Cannot deserialize class - not found: " + name); } return clazz; } private String nameForClass(Class clazz) { var name = classToName.get(clazz); if (name == null) { throw new IllegalArgumentException( "Cannot serialize class - not found: " + clazz.getName()); } return name; } } static final class ZonedDateTimeTypeAdapter extends TypeAdapter { @Override public void write(final JsonWriter out, final ZonedDateTime value) throws IOException { out.value(value.format(DateTimeFormatter.ISO_ZONED_DATE_TIME)); } @Override public ZonedDateTime read(final JsonReader in) throws IOException { return ZonedDateTime.parse(in.nextString(), DateTimeFormatter.ISO_ZONED_DATE_TIME); } } static final class LocalDateTimeTypeAdapter extends TypeAdapter { @Override public void write(JsonWriter out, LocalDateTime value) throws IOException { out.value(value.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); } @Override public LocalDateTime read(JsonReader in) throws IOException { return LocalDateTime.parse(in.nextString(), DateTimeFormatter.ISO_LOCAL_DATE_TIME); } } static final class InstantTypeAdapter extends TypeAdapter { @Override public void write(JsonWriter out, Instant value) throws IOException { out.value(DateTimeFormatter.ISO_INSTANT.format(value)); } @Override public Instant read(JsonReader in) throws IOException { return DateTimeFormatter.ISO_INSTANT.parse(in.nextString(), Instant::from); } } static final class DurationTypeAdapter extends TypeAdapter { @Override public void write(JsonWriter out, Duration value) throws IOException { out.value(value.get(ChronoUnit.SECONDS)); } @Override public Duration read(JsonReader in) throws IOException { return Duration.of(in.nextLong(), ChronoUnit.SECONDS); } } static final class LocalDateTypeAdapter extends TypeAdapter { @Override public void write(JsonWriter out, LocalDate value) throws IOException { out.value(DateTimeFormatter.ISO_LOCAL_DATE.format(value)); } @Override public LocalDate read(JsonReader in) throws IOException { return DateTimeFormatter.ISO_LOCAL_DATE.parse(in.nextString(), LocalDate::from); } } static final class MonthDayTypeAdapter extends TypeAdapter { private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("d/M"); @Override public void write(JsonWriter out, MonthDay value) throws IOException { out.value(value.format(formatter)); } @Override public MonthDay read(JsonReader in) throws IOException { return MonthDay.parse(in.nextString(), formatter); } } static final class PeriodTypeAdapter extends TypeAdapter { @Override public void write(JsonWriter out, Period value) throws IOException { out.value(value.toString()); } @Override public Period read(JsonReader in) throws IOException { return Period.parse(in.nextString()); } } static final class YearTypeAdapter extends TypeAdapter { @Override public void write(JsonWriter out, Year value) throws IOException { out.value(value.getValue()); } @Override public Year read(JsonReader in) throws IOException { return Year.of(in.nextInt()); } } static final class YearMonthAdapter extends TypeAdapter { @Override public void write(JsonWriter out, YearMonth value) throws IOException { out.value(value.toString()); } @Override public YearMonth read(JsonReader in) throws IOException { return YearMonth.parse(in.nextString()); } } static final class UtcDateTypeAdapter extends TypeAdapter { private final TimeZone UTC_TIME_ZONE = TimeZone.getTimeZone("UTC"); @Override public void write(JsonWriter out, Date date) throws IOException { if (date == null) { out.nullValue(); } else { String value = format(date, true, UTC_TIME_ZONE); out.value(value); } } @Override public Date read(JsonReader in) throws IOException { try { if (in.peek() == JsonToken.NULL) { in.nextNull(); return null; } String date = in.nextString(); // Instead of using iso8601Format.parse(value), we use Jackson's date parsing // This is because Android doesn't support XXX because it is JDK 1.6 return parse(date, new ParsePosition(0)); } catch (ParseException e) { throw new JsonParseException(e); } } // Date parsing code from Jackson databind ISO8601Utils.java // https://github.com/FasterXML/jackson-databind/blob/master/src/main/java/com/fasterxml/jackson/databind/util/ISO8601Utils.java private static final String GMT_ID = "GMT"; /** * Format date into yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm] * * @param date the date to format * @param millis true to include millis precision otherwise false * @param tz timezone to use for the formatting (GMT will produce 'Z') * @return the date formatted as yyyy-MM-ddThh:mm:ss[.sss][Z|[+-]hh:mm] */ private static String format(Date date, boolean millis, TimeZone tz) { Calendar calendar = new GregorianCalendar(tz, Locale.US); calendar.setTime(date); // estimate capacity of buffer as close as we can (yeah, that's pedantic ;) int capacity = "yyyy-MM-ddThh:mm:ss".length(); capacity += millis ? ".sss".length() : 0; capacity += tz.getRawOffset() == 0 ? "Z".length() : "+hh:mm".length(); StringBuilder formatted = new StringBuilder(capacity); padInt(formatted, calendar.get(Calendar.YEAR), "yyyy".length()); formatted.append('-'); padInt(formatted, calendar.get(Calendar.MONTH) + 1, "MM".length()); formatted.append('-'); padInt(formatted, calendar.get(Calendar.DAY_OF_MONTH), "dd".length()); formatted.append('T'); padInt(formatted, calendar.get(Calendar.HOUR_OF_DAY), "hh".length()); formatted.append(':'); padInt(formatted, calendar.get(Calendar.MINUTE), "mm".length()); formatted.append(':'); padInt(formatted, calendar.get(Calendar.SECOND), "ss".length()); if (millis) { formatted.append('.'); padInt(formatted, calendar.get(Calendar.MILLISECOND), "sss".length()); } int offset = tz.getOffset(calendar.getTimeInMillis()); if (offset != 0) { int hours = Math.abs((offset / (60 * 1000)) / 60); int minutes = Math.abs((offset / (60 * 1000)) % 60); formatted.append(offset < 0 ? '-' : '+'); padInt(formatted, hours, "hh".length()); formatted.append(':'); padInt(formatted, minutes, "mm".length()); } else { formatted.append('Z'); } return formatted.toString(); } /** * Zero pad a number to a specified length * * @param buffer buffer to use for padding * @param value the integer value to pad if necessary. * @param length the length of the string we should zero pad */ private static void padInt(StringBuilder buffer, int value, int length) { String strValue = Integer.toString(value); buffer.append("0".repeat(Math.max(0, length - strValue.length()))); buffer.append(strValue); } /** * Parse a date from ISO-8601 formatted string. It expects a format * [yyyy-MM-dd|yyyyMMdd][T(hh:mm[:ss[.sss]]|hhmm[ss[.sss]])]?[Z|[+-]hh:mm]] * * @param date ISO string to parse in the appropriate format. * @param pos The position to start parsing from, updated to where parsing stopped. * @return the parsed date * @throws ParseException if the date is not in the appropriate format */ private static Date parse(String date, ParsePosition pos) throws ParseException { Exception fail; try { int offset = pos.getIndex(); // extract year int year = parseInt(date, offset, offset += 4); if (checkOffset(date, offset, '-')) { offset += 1; } // extract month int month = parseInt(date, offset, offset += 2); if (checkOffset(date, offset, '-')) { offset += 1; } // extract day int day = parseInt(date, offset, offset += 2); // default time value int hour = 0; int minutes = 0; int seconds = 0; int milliseconds = 0; // always use 0 otherwise returned date will include millis of current time if (checkOffset(date, offset, 'T')) { // extract hours, minutes, seconds and milliseconds hour = parseInt(date, offset += 1, offset += 2); if (checkOffset(date, offset, ':')) { offset += 1; } minutes = parseInt(date, offset, offset += 2); if (checkOffset(date, offset, ':')) { offset += 1; } // second and milliseconds can be optional if (date.length() > offset) { char c = date.charAt(offset); if (c != 'Z' && c != '+' && c != '-') { seconds = parseInt(date, offset, offset += 2); // milliseconds can be optional in the format if (checkOffset(date, offset, '.')) { milliseconds = parseInt(date, offset += 1, offset += 3); } } } } // extract timezone String timezoneId; if (date.length() <= offset) { throw new IllegalArgumentException("No time zone indicator"); } char timezoneIndicator = date.charAt(offset); if (timezoneIndicator == '+' || timezoneIndicator == '-') { String timezoneOffset = date.substring(offset); timezoneId = GMT_ID + timezoneOffset; offset += timezoneOffset.length(); } else if (timezoneIndicator == 'Z') { timezoneId = GMT_ID; offset += 1; } else { throw new IndexOutOfBoundsException("Invalid time zone indicator " + timezoneIndicator); } TimeZone timezone = TimeZone.getTimeZone(timezoneId); if (!timezone.getID().equals(timezoneId)) { throw new IndexOutOfBoundsException(); } Calendar calendar = new GregorianCalendar(timezone); calendar.setLenient(false); calendar.set(Calendar.YEAR, year); calendar.set(Calendar.MONTH, month - 1); calendar.set(Calendar.DAY_OF_MONTH, day); calendar.set(Calendar.HOUR_OF_DAY, hour); calendar.set(Calendar.MINUTE, minutes); calendar.set(Calendar.SECOND, seconds); calendar.set(Calendar.MILLISECOND, milliseconds); pos.setIndex(offset); return calendar.getTime(); // If we get a ParseException it'll already have the right message/offset. // Other exception types can convert here. } catch (IndexOutOfBoundsException | IllegalArgumentException e) { fail = e; } String input = (date == null) ? null : ("'" + date + "'"); throw new ParseException( "Failed to parse date [" + input + "]: " + fail.getMessage(), pos.getIndex()); } /** * Check if the expected character exist at the given offset in the value. * * @param value the string to check at the specified offset * @param offset the offset to look for the expected character * @param expected the expected character * @return true if the expected character exist at the given offset */ private static boolean checkOffset(String value, int offset, char expected) { return (offset < value.length()) && (value.charAt(offset) == expected); } /** * Parse an integer located between 2 given offsets in a string * * @param value the string to parse * @param beginIndex the start index for the integer in the string * @param endIndex the end index for the integer in the string * @return the int * @throws NumberFormatException if the value is not a number */ private static int parseInt(String value, int beginIndex, int endIndex) throws NumberFormatException { if (beginIndex < 0 || endIndex > value.length() || beginIndex > endIndex) { throw new NumberFormatException(value); } // use same logic as in Integer.parseInt() but less generic we're not supporting negative // values int i = beginIndex; int result = 0; int digit; if (i < endIndex) { digit = Character.digit(value.charAt(i++), 10); if (digit < 0) { throw new NumberFormatException("Invalid number: " + value); } result = -digit; } while (i < endIndex) { digit = Character.digit(value.charAt(i++), 10); if (digit < 0) { throw new NumberFormatException("Invalid number: " + value); } result *= 10; result -= digit; } return -result; } } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/DefaultMigrationManager.java ================================================ package com.gruelbox.transactionoutbox; import static com.gruelbox.transactionoutbox.spi.Utils.uncheck; import java.io.PrintWriter; import java.io.Writer; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; /** * Simple database migration manager. Inspired by Flyway, Liquibase, Morf etc, just trimmed down for * minimum dependencies. */ @Slf4j class DefaultMigrationManager { private static final Executor basicExecutor = runnable -> { new Thread(runnable).start(); }; private static CountDownLatch waitLatch; private static CountDownLatch readyLatch; static void withLatch(CountDownLatch readyLatch, Consumer runnable) { waitLatch = new CountDownLatch(1); DefaultMigrationManager.readyLatch = readyLatch; try { runnable.accept(waitLatch); } finally { waitLatch = null; DefaultMigrationManager.readyLatch = null; } } static void migrate(TransactionManager transactionManager, Dialect dialect) { transactionManager.inTransaction( transaction -> { try { int currentVersion = currentVersion(transaction.connection(), dialect); dialect .getMigrations() .filter(migration -> migration.getVersion() > currentVersion) .forEach( migration -> uncheck( () -> runSql(transactionManager, transaction.connection(), migration))); } catch (Exception e) { throw new RuntimeException("Migrations failed", e); } }); } static void writeSchema(Writer writer, Dialect dialect) { PrintWriter printWriter = new PrintWriter(writer); dialect .getMigrations() .forEach( migration -> { printWriter.print("-- "); printWriter.print(migration.getVersion()); printWriter.print(": "); printWriter.println(migration.getName()); if (migration.getSql() == null || migration.getSql().isEmpty()) { printWriter.println("-- Nothing for " + dialect); } else { printWriter.println(migration.getSql()); } printWriter.println(); }); printWriter.flush(); } private static void runSql(TransactionManager txm, Connection connection, Migration migration) throws SQLException { log.info("Running migration {}: {}", migration.getVersion(), migration.getName()); if (migration.getSql() != null && !migration.getSql().isEmpty()) { CompletableFuture.runAsync( () -> { try { txm.inTransactionThrows( tx -> { try (var s = tx.connection().prepareStatement(migration.getSql())) { s.execute(); } }); } catch (SQLException e) { throw new RuntimeException(e); } }, basicExecutor) .join(); } try (var s = connection.prepareStatement("UPDATE TXNO_VERSION SET version = ?")) { s.setInt(1, migration.getVersion()); if (s.executeUpdate() != 1) { throw new IllegalStateException("Version table should already exist"); } } } private static int currentVersion(Connection connection, Dialect dialect) throws SQLException { dialect.createVersionTableIfNotExists(connection); int version = fetchCurrentVersion(connection, dialect); if (version >= 0) { return version; } try { log.info("No version record found. Attempting to create"); if (waitLatch != null) { log.info("Stopping at latch"); readyLatch.countDown(); if (!waitLatch.await(10, TimeUnit.SECONDS)) { throw new IllegalStateException("Latch not released in 10 seconds"); } log.info("Latch released"); } try (var s = connection.prepareStatement("INSERT INTO TXNO_VERSION (version) VALUES (0)")) { s.executeUpdate(); } log.info("Created version record."); return fetchCurrentVersion(connection, dialect); } catch (Exception e) { log.info( "Error attempting to create ({} - {}). May have been beaten to it, attempting second fetch", e.getClass().getSimpleName(), e.getMessage()); version = fetchCurrentVersion(connection, dialect); if (version >= 0) { return version; } throw new IllegalStateException("Unable to read or create version record", e); } } private static int fetchCurrentVersion(Connection connection, Dialect dialect) throws SQLException { try (PreparedStatement s = connection.prepareStatement(dialect.getFetchCurrentVersion()); ResultSet rs = s.executeQuery()) { if (rs.next()) { var version = rs.getInt(1); log.info("Current version is {}, obtained lock", version); if (rs.next()) { throw new IllegalStateException("More than one version record"); } return version; } return -1; } } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/DefaultPersistor.java ================================================ package com.gruelbox.transactionoutbox; import static java.nio.charset.StandardCharsets.UTF_8; import java.io.*; import java.sql.*; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; /** * The default {@link Persistor} for {@link TransactionOutbox}. * *

Saves requests to a relational database table, by default called {@code TXNO_OUTBOX}. This can * optionally be automatically created and upgraded by {@link DefaultPersistor}, although this * behaviour can be disabled if you wish. * *

More significant changes can be achieved by subclassing, which is explicitly supported. If, on * the other hand, you want to use a completely non-relational underlying data store or do something * equally esoteric, you may prefer to implement {@link Persistor} from the ground up. */ @Slf4j @SuperBuilder @AllArgsConstructor(access = AccessLevel.PROTECTED) public class DefaultPersistor implements Persistor, Validatable { private static final String ALL_FIELDS = "id, uniqueRequestId, invocation, topic, seq, lastAttemptTime, nextAttemptTime, attempts, blocked, processed, version"; /** * @param writeLockTimeoutSeconds How many seconds to wait before timing out on obtaining a write * lock. There's no point making this long; it's always better to just back off as quickly as * possible and try another record. Generally these lock timeouts only kick in if the database * does not support skip locking. */ @SuppressWarnings("JavaDoc") @Builder.Default private final int writeLockTimeoutSeconds = 2; /** * @param dialect The database dialect to use. Required. */ @SuppressWarnings("JavaDoc") private final Dialect dialect; /** * @param tableName The database table name. The default is {@code TXNO_OUTBOX}. */ @SuppressWarnings("JavaDoc") @Builder.Default private final String tableName = "TXNO_OUTBOX"; /** * @param migrate Set to false to disable automatic database migrations. This may be preferred if * the default migration behaviour interferes with your existing toolset, and you prefer to * manage the migrations explicitly (e.g. using FlyWay or Liquibase), or you do not give the * application DDL permissions at runtime. You may use {@link #writeSchema(Writer)} to access * the migrations. */ @SuppressWarnings("JavaDoc") @Builder.Default private final boolean migrate = true; /** * @param serializer The serializer to use for {@link Invocation}s. See {@link * InvocationSerializer} for more information. Defaults to {@link * InvocationSerializer#createDefaultJsonSerializer()} with no custom serializable classes. */ @SuppressWarnings("JavaDoc") @Builder.Default private final InvocationSerializer serializer = InvocationSerializer.createDefaultJsonSerializer(); @Override public void validate(Validator validator) { validator.notNull("dialect", dialect); validator.notNull("tableName", tableName); } @Override public void migrate(TransactionManager transactionManager) { if (migrate) { DefaultMigrationManager.migrate(transactionManager, dialect); } } /** * Provides access to the database schema so that you may optionally use your existing toolset to * manage migrations. * * @param writer The writer to which the migrations are written. */ public void writeSchema(Writer writer) { DefaultMigrationManager.writeSchema(writer, dialect); } @Override public void save(Transaction tx, TransactionOutboxEntry entry) throws SQLException, AlreadyScheduledException { var insertSql = "INSERT INTO " + tableName + " (" + ALL_FIELDS + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; var writer = new StringWriter(); serializer.serializeInvocation(entry.getInvocation(), writer); if (entry.getTopic() != null) { setNextSequence(tx, entry); log.info("Assigned sequence number {} to topic {}", entry.getSequence(), entry.getTopic()); } PreparedStatement stmt = tx.prepareBatchStatement(insertSql); setupInsert(entry, writer, stmt); if (entry.getUniqueRequestId() == null) { stmt.addBatch(); log.debug("Inserted {} in batch", entry.description()); } else { try { stmt.executeUpdate(); log.debug("Inserted {} immediately", entry.description()); } catch (Exception e) { if (indexViolation(e)) { throw new AlreadyScheduledException( "Request " + entry.description() + " already exists", e); } throw e; } } } @Override public Invocation serializeAndDeserialize(Invocation invocation) { try (var baos = new ByteArrayOutputStream()) { try (var writer = new OutputStreamWriter(baos, UTF_8)) { serializer.serializeInvocation(invocation, writer); } ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); try (Reader reader = new InputStreamReader(bais, UTF_8)) { return serializer.deserializeInvocation(reader); } } catch (IOException e) { throw new UncheckedException(e); } } private void setNextSequence(Transaction tx, TransactionOutboxEntry entry) throws SQLException { //noinspection resource var seqSelect = tx.prepareBatchStatement(dialect.getFetchNextSequence()); seqSelect.setString(1, entry.getTopic()); try (ResultSet rs = seqSelect.executeQuery()) { if (rs.next()) { entry.setSequence(rs.getLong(1) + 1L); //noinspection resource var seqUpdate = tx.prepareBatchStatement("UPDATE TXNO_SEQUENCE SET seq = ? WHERE topic = ?"); seqUpdate.setLong(1, entry.getSequence()); seqUpdate.setString(2, entry.getTopic()); seqUpdate.executeUpdate(); } else { try { entry.setSequence(1L); //noinspection resource var seqInsert = tx.prepareBatchStatement("INSERT INTO TXNO_SEQUENCE (topic, seq) VALUES (?, ?)"); seqInsert.setString(1, entry.getTopic()); seqInsert.setLong(2, entry.getSequence()); seqInsert.executeUpdate(); } catch (Exception e) { if (indexViolation(e)) { setNextSequence(tx, entry); } else { throw e; } } } } } private boolean indexViolation(Exception e) { return (e instanceof SQLIntegrityConstraintViolationException) || (e.getClass().getName().equals("org.postgresql.util.PSQLException") && e.getMessage().contains("constraint")) || (e.getClass().getName().equals("com.microsoft.sqlserver.jdbc.SQLServerException") && e.getMessage().contains("duplicate key")); } private void setupInsert( TransactionOutboxEntry entry, StringWriter writer, PreparedStatement stmt) throws SQLException { stmt.setString(1, entry.getId()); stmt.setString(2, entry.getUniqueRequestId()); stmt.setString(3, writer.toString()); stmt.setString(4, entry.getTopic() == null ? "*" : entry.getTopic()); if (entry.getSequence() == null) { stmt.setObject(5, null); } else { stmt.setLong(5, entry.getSequence()); } stmt.setTimestamp( 6, entry.getLastAttemptTime() == null ? null : Timestamp.from(entry.getLastAttemptTime())); stmt.setTimestamp(7, Timestamp.from(entry.getNextAttemptTime())); stmt.setInt(8, entry.getAttempts()); stmt.setBoolean(9, entry.isBlocked()); stmt.setBoolean(10, entry.isProcessed()); stmt.setInt(11, entry.getVersion()); } @Override public void delete(Transaction tx, TransactionOutboxEntry entry) throws Exception { //noinspection resource try (PreparedStatement stmt = tx.connection().prepareStatement(dialect.getDelete().replace("{{table}}", tableName))) { stmt.setString(1, entry.getId()); stmt.setInt(2, entry.getVersion()); if (stmt.executeUpdate() != 1) { throw new OptimisticLockException(); } log.debug("Deleted {}", entry.description()); } } @Override public void update(Transaction tx, TransactionOutboxEntry entry) throws Exception { //noinspection resource try (PreparedStatement stmt = tx.connection() .prepareStatement( // language=MySQL "UPDATE " + tableName + " " + "SET lastAttemptTime = ?, nextAttemptTime = ?, attempts = ?, blocked = ?, processed = ?, version = ? " + "WHERE id = ? and version = ?")) { stmt.setTimestamp( 1, entry.getLastAttemptTime() == null ? null : Timestamp.from(entry.getLastAttemptTime())); stmt.setTimestamp(2, Timestamp.from(entry.getNextAttemptTime())); stmt.setInt(3, entry.getAttempts()); stmt.setBoolean(4, entry.isBlocked()); stmt.setBoolean(5, entry.isProcessed()); stmt.setInt(6, entry.getVersion() + 1); stmt.setString(7, entry.getId()); stmt.setInt(8, entry.getVersion()); if (stmt.executeUpdate() != 1) { throw new OptimisticLockException(); } entry.setVersion(entry.getVersion() + 1); log.debug("Updated {}", entry.description()); } } @Override public boolean lock(Transaction tx, TransactionOutboxEntry entry) throws Exception { //noinspection resource try (PreparedStatement stmt = tx.connection() .prepareStatement( dialect .getLock() .replace("{{table}}", tableName) .replace("{{allFields}}", ALL_FIELDS))) { stmt.setString(1, entry.getId()); stmt.setInt(2, entry.getVersion()); stmt.setQueryTimeout(writeLockTimeoutSeconds); try { try (ResultSet rs = stmt.executeQuery()) { if (!rs.next()) { return false; } // Ensure that subsequent processing uses a deserialized invocation rather than // the object from the caller, which might not serialize well and thus cause a // difference between immediate and retry processing try (Reader invocationStream = rs.getCharacterStream("invocation")) { entry.setInvocation(serializer.deserializeInvocation(invocationStream)); } return true; } } catch (SQLTimeoutException e) { log.debug("Lock attempt timed out on {}", entry.description()); return false; } } } @Override public boolean unblock(Transaction tx, String entryId) throws Exception { //noinspection resource try (PreparedStatement stmt = tx.connection() .prepareStatement( "UPDATE " + tableName + " SET attempts = 0, blocked = " + dialect.booleanValue(false) + " " + "WHERE blocked = " + dialect.booleanValue(true) + " AND processed = " + dialect.booleanValue(false) + " AND id = ?")) { stmt.setString(1, entryId); stmt.setQueryTimeout(writeLockTimeoutSeconds); return stmt.executeUpdate() != 0; } } @Override public List selectBatch(Transaction tx, int batchSize, Instant now) throws Exception { //noinspection resource try (PreparedStatement stmt = tx.connection() .prepareStatement( dialect .getSelectBatch() .replace("{{table}}", tableName) .replace("{{batchSize}}", Integer.toString(batchSize)) .replace("{{allFields}}", ALL_FIELDS))) { stmt.setTimestamp(1, Timestamp.from(now)); var result = new ArrayList(batchSize); gatherResults(stmt, result); return result; } } @Override public Collection selectNextInTopics( Transaction tx, int batchSize, Instant now) throws Exception { var sql = dialect .getFetchNextInAllTopics() .replace("{{table}}", tableName) .replace("{{batchSize}}", Integer.toString(batchSize)) .replace("{{allFields}}", ALL_FIELDS); //noinspection resource try (PreparedStatement stmt = tx.connection().prepareStatement(sql)) { stmt.setTimestamp(1, Timestamp.from(now)); var results = new ArrayList(); gatherResults(stmt, results); return results; } } @Override public Collection selectNextInSelectedTopics( Transaction tx, List topicNames, int batchSize, Instant now) throws Exception { var topicsInParameterList = topicNames.stream().map(it -> "?").collect(Collectors.joining(",")); var sql = dialect .getFetchNextInSelectedTopics() .replace("{{table}}", tableName) .replace("{{topicNames}}", topicsInParameterList) .replace("{{batchSize}}", Integer.toString(batchSize)) .replace("{{allFields}}", ALL_FIELDS); //noinspection resource try (PreparedStatement stmt = tx.connection().prepareStatement(sql)) { var counter = 1; for (var topicName : topicNames) { stmt.setString(counter, topicName); counter++; } stmt.setTimestamp(counter, Timestamp.from(now)); var results = new ArrayList(); gatherResults(stmt, results); return results; } } @Override public int deleteProcessedAndExpired(Transaction tx, int batchSize, Instant now) throws Exception { //noinspection resource try (PreparedStatement stmt = tx.connection() .prepareStatement( dialect .getDeleteExpired() .replace("{{table}}", tableName) .replace("{{batchSize}}", Integer.toString(batchSize)))) { stmt.setTimestamp(1, Timestamp.from(now)); return stmt.executeUpdate(); } } private void gatherResults(PreparedStatement stmt, Collection output) throws SQLException, IOException { try (ResultSet rs = stmt.executeQuery()) { while (rs.next()) { output.add(map(rs)); } log.debug("Found {} results", output.size()); } } private TransactionOutboxEntry map(ResultSet rs) throws SQLException, IOException { String topic = rs.getString("topic"); Long sequence = rs.getLong("seq"); if (rs.wasNull()) { sequence = null; } // Reading invocationStream *must* occur first because some drivers (ex. SQL Server) // implement true streams that are not buffered in memory. Calling any other getter // on ResultSet before invocationStream is read will cause Reader to be closed // prematurely. try (Reader invocationStream = rs.getCharacterStream("invocation")) { Invocation invocation; try { invocation = serializer.deserializeInvocation(invocationStream); } catch (IOException e) { invocation = new FailedDeserializingInvocation(e); } TransactionOutboxEntry entry = TransactionOutboxEntry.builder() .invocation(invocation) .id(rs.getString("id")) .uniqueRequestId(rs.getString("uniqueRequestId")) .topic("*".equals(topic) ? null : topic) .sequence(sequence) .lastAttemptTime( rs.getTimestamp("lastAttemptTime") == null ? null : rs.getTimestamp("lastAttemptTime").toInstant()) .nextAttemptTime(rs.getTimestamp("nextAttemptTime").toInstant()) .attempts(rs.getInt("attempts")) .blocked(rs.getBoolean("blocked")) .processed(rs.getBoolean("processed")) .version(rs.getInt("version")) .build(); log.debug("Found {}", entry); return entry; } } // For testing. Assumed low volume. @Override public void clear(Transaction tx) throws SQLException { //noinspection resource try (Statement stmt = tx.connection().createStatement()) { stmt.execute("DELETE FROM " + tableName); } } @Override public boolean checkConnection(Transaction tx) throws SQLException { //noinspection resource try (Statement stmt = tx.connection().createStatement(); ResultSet rs = stmt.executeQuery(dialect.getCheckSql())) { return rs.next() && (rs.getInt(1) == 1); } } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/Dialect.java ================================================ package com.gruelbox.transactionoutbox; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; import java.util.stream.Stream; /** The SQL dialects supported by {@link DefaultPersistor}. */ public interface Dialect { String getDelete(); /** * @return Format string for the SQL required to delete expired retained records. */ String getDeleteExpired(); String getSelectBatch(); String getLock(); String getCheckSql(); String getFetchNextInAllTopics(); String getFetchNextInSelectedTopics(); String getFetchCurrentVersion(); String getFetchNextSequence(); String booleanValue(boolean criteriaValue); void createVersionTableIfNotExists(Connection connection) throws SQLException; Stream getMigrations(); Dialect MY_SQL_5 = DefaultDialect.builder("MY_SQL_5") .changeMigration( 13, "ALTER TABLE TXNO_OUTBOX MODIFY COLUMN invocation mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci") .build(); Dialect MY_SQL_8 = DefaultDialect.builder("MY_SQL_8") .fetchNextInAllTopics( "WITH raw AS(SELECT {{allFields}}, (ROW_NUMBER() OVER(PARTITION BY topic ORDER BY seq)) as rn" + " FROM {{table}} WHERE processed = false AND topic <> '*')" + " SELECT * FROM raw WHERE rn = 1 AND nextAttemptTime < ? LIMIT {{batchSize}}") .fetchNextInSelectedTopics( "WITH raw AS(SELECT {{allFields}}, (ROW_NUMBER() OVER(PARTITION BY topic ORDER BY seq)) as rn" + " FROM {{table}} WHERE processed = false AND topic IN ({{topicNames}}))" + " SELECT * FROM raw WHERE rn = 1 AND nextAttemptTime < ? LIMIT {{batchSize}}") .deleteExpired( "DELETE FROM {{table}} WHERE nextAttemptTime < ? AND processed = true AND blocked = false" + " LIMIT {{batchSize}}") .selectBatch( "SELECT {{allFields}} FROM {{table}} WHERE nextAttemptTime < ? " + "AND blocked = false AND processed = false AND topic = '*' LIMIT {{batchSize}} FOR UPDATE " + "SKIP LOCKED") .lock( "SELECT id, invocation FROM {{table}} WHERE id = ? AND version = ? FOR " + "UPDATE SKIP LOCKED") .changeMigration( 13, "ALTER TABLE TXNO_OUTBOX MODIFY COLUMN invocation mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci") .build(); Dialect POSTGRESQL_9 = DefaultDialect.builder("POSTGRESQL_9") .fetchNextInAllTopics( "WITH raw AS(SELECT {{allFields}}, (ROW_NUMBER() OVER(PARTITION BY topic ORDER BY seq)) as rn" + " FROM {{table}} WHERE processed = false AND topic <> '*')" + " SELECT * FROM raw WHERE rn = 1 AND nextAttemptTime < ? LIMIT {{batchSize}}") .fetchNextInSelectedTopics( "WITH raw AS(SELECT {{allFields}}, (ROW_NUMBER() OVER(PARTITION BY topic ORDER BY seq)) as rn" + " FROM {{table}} WHERE processed = false AND topic IN ({{topicNames}}))" + " SELECT * FROM raw WHERE rn = 1 AND nextAttemptTime < ? LIMIT {{batchSize}}") .deleteExpired( "DELETE FROM {{table}} WHERE id IN " + "(SELECT id FROM {{table}} WHERE nextAttemptTime < ? AND processed = true AND blocked = false LIMIT {{batchSize}})") .selectBatch( "SELECT {{allFields}} FROM {{table}} WHERE nextAttemptTime < ? " + "AND blocked = false AND processed = false AND topic = '*' LIMIT " + "{{batchSize}} FOR UPDATE SKIP LOCKED") .lock( "SELECT id, invocation FROM {{table}} WHERE id = ? AND version = ? FOR " + "UPDATE SKIP LOCKED") .changeMigration( 5, "ALTER TABLE TXNO_OUTBOX ALTER COLUMN uniqueRequestId TYPE VARCHAR(250)") .changeMigration(6, "ALTER TABLE TXNO_OUTBOX RENAME COLUMN blacklisted TO blocked") .changeMigration(7, "ALTER TABLE TXNO_OUTBOX ADD COLUMN lastAttemptTime TIMESTAMP(6)") .disableMigration(8) .build(); Dialect H2 = DefaultDialect.builder("H2") .changeMigration(5, "ALTER TABLE TXNO_OUTBOX ALTER COLUMN uniqueRequestId VARCHAR(250)") .changeMigration(6, "ALTER TABLE TXNO_OUTBOX RENAME COLUMN blacklisted TO blocked") .disableMigration(8) .build(); Dialect ORACLE = DefaultDialect.builder("ORACLE") .fetchNextInAllTopics( "WITH cte1 AS (SELECT {{allFields}}, (ROW_NUMBER() OVER(PARTITION BY topic ORDER BY seq)) as rn" + " FROM {{table}} WHERE processed = 0 AND topic <> '*')" + " SELECT * FROM cte1 WHERE rn = 1 AND nextAttemptTime < ? AND ROWNUM <= {{batchSize}}") .fetchNextInSelectedTopics( "WITH cte1 AS (SELECT {{allFields}}, (ROW_NUMBER() OVER(PARTITION BY topic ORDER BY seq)) as rn" + " FROM {{table}} WHERE processed = 0 AND topic IN ({{topicNames}}))" + " SELECT * FROM cte1 WHERE rn = 1 AND nextAttemptTime < ? AND ROWNUM <= {{batchSize}}") .deleteExpired( "DELETE FROM {{table}} WHERE nextAttemptTime < ? AND processed = 1 AND blocked = 0 " + "AND ROWNUM <= {{batchSize}}") .selectBatch( "SELECT {{allFields}} FROM {{table}} WHERE nextAttemptTime < ? " + "AND blocked = 0 AND processed = 0 AND topic = '*' AND ROWNUM <= {{batchSize}} FOR UPDATE " + "SKIP LOCKED") .lock( "SELECT id, invocation FROM {{table}} WHERE id = ? AND version = ? FOR " + "UPDATE SKIP LOCKED") .checkSql("SELECT 1 FROM DUAL") .changeMigration( 1, "CREATE TABLE TXNO_OUTBOX (\n" + " id VARCHAR2(36) PRIMARY KEY,\n" + " invocation CLOB,\n" + " nextAttemptTime TIMESTAMP(6),\n" + " attempts NUMBER,\n" + " blacklisted NUMBER(1),\n" + " version NUMBER\n" + ")") .changeMigration( 2, "ALTER TABLE TXNO_OUTBOX ADD uniqueRequestId VARCHAR(100) NULL UNIQUE") .changeMigration(3, "ALTER TABLE TXNO_OUTBOX ADD processed NUMBER(1)") .changeMigration(5, "ALTER TABLE TXNO_OUTBOX MODIFY uniqueRequestId VARCHAR2(250)") .changeMigration(6, "ALTER TABLE TXNO_OUTBOX RENAME COLUMN blacklisted TO blocked") .changeMigration(7, "ALTER TABLE TXNO_OUTBOX ADD lastAttemptTime TIMESTAMP(6)") .disableMigration(8) .changeMigration(9, "ALTER TABLE TXNO_OUTBOX ADD topic VARCHAR(250) DEFAULT '*' NOT NULL") .changeMigration(10, "ALTER TABLE TXNO_OUTBOX ADD seq NUMBER") .changeMigration( 11, "CREATE TABLE TXNO_SEQUENCE (topic VARCHAR(250) NOT NULL, seq NUMBER NOT NULL, CONSTRAINT PK_TXNO_SEQUENCE PRIMARY KEY (topic, seq))") .booleanValueFrom(v -> v ? "1" : "0") .createVersionTableBy( connection -> { try (Statement s = connection.createStatement()) { try { s.execute("CREATE TABLE TXNO_VERSION (version NUMBER)"); } catch (SQLException e) { // oracle code for name already used by an existing object if (!e.getMessage().contains("955")) { throw e; } } } }) .build(); Dialect MS_SQL_SERVER = DefaultDialect.builder("MS_SQL_SERVER") .lock( "SELECT id, invocation FROM {{table}} WITH (UPDLOCK, ROWLOCK, READPAST) WHERE id = ? AND version = ?") .selectBatch( "SELECT TOP ({{batchSize}}) {{allFields}} FROM {{table}} " + "WITH (UPDLOCK, ROWLOCK, READPAST) WHERE nextAttemptTime < ? AND topic = '*' " + "AND blocked = 0 AND processed = 0") .delete("DELETE FROM {{table}} WITH (ROWLOCK, READPAST) WHERE id = ? and version = ?") .deleteExpired( "DELETE TOP ({{batchSize}}) FROM {{table}} " + "WHERE nextAttemptTime < ? AND processed = 1 AND blocked = 0") .fetchCurrentVersion("SELECT version FROM TXNO_VERSION WITH (UPDLOCK, ROWLOCK, READPAST)") .fetchNextInAllTopics( "SELECT TOP {{batchSize}} {{allFields}} FROM {{table}} a" + " WHERE processed = 0 AND topic <> '*' AND nextAttemptTime < ?" + " AND seq = (" + "SELECT MIN(seq) FROM {{table}} b WHERE b.topic=a.topic AND b.processed = 0" + ")") .fetchNextInSelectedTopics( "SELECT TOP {{batchSize}} {{allFields}} FROM {{table}} a" + " WHERE processed = 0 AND topic IN ({{topicNames}}) AND nextAttemptTime < ?" + " AND seq = (" + "SELECT MIN(seq) FROM {{table}} b WHERE b.topic=a.topic AND b.processed = 0" + ")") .fetchNextSequence( "SELECT seq FROM TXNO_SEQUENCE WITH (UPDLOCK, ROWLOCK, READPAST) WHERE topic = ?") .booleanValueFrom(v -> v ? "1" : "0") .changeMigration( 1, "CREATE TABLE TXNO_OUTBOX (\n" + " id VARCHAR(36) PRIMARY KEY,\n" + " invocation NVARCHAR(MAX),\n" + " nextAttemptTime DATETIME2(6),\n" + " attempts INT,\n" + " blocked BIT,\n" + " version INT,\n" + " uniqueRequestId VARCHAR(250),\n" + " processed BIT,\n" + " lastAttemptTime DATETIME2(6),\n" + " topic VARCHAR(250) DEFAULT '*' NOT NULL,\n" + " seq INT\n" + ")") .disableMigration(2) .disableMigration(3) .changeMigration( 4, "CREATE INDEX IX_TXNO_OUTBOX_1 ON TXNO_OUTBOX (processed, blocked, nextAttemptTime)") .disableMigration(5) .disableMigration(6) .disableMigration(7) .changeMigration( 8, "CREATE UNIQUE INDEX UX_TXNO_OUTBOX_uniqueRequestId ON TXNO_OUTBOX (uniqueRequestId) WHERE uniqueRequestId IS NOT NULL") .disableMigration(9) .disableMigration(10) .changeMigration( 11, "CREATE TABLE TXNO_SEQUENCE (topic VARCHAR(250) NOT NULL, seq INT NOT NULL, CONSTRAINT " + "PK_TXNO_SEQUENCE PRIMARY KEY (topic, seq))") .createVersionTableBy( connection -> { try (Statement s = connection.createStatement()) { s.execute( "IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'TXNO_VERSION')\n" + "BEGIN\n" + " CREATE TABLE TXNO_VERSION (\n" + " version INT\n" + " );" + "END"); } }) .build(); } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/DriverConnectionProvider.java ================================================ package com.gruelbox.transactionoutbox; import static com.gruelbox.transactionoutbox.spi.Utils.uncheckedly; import java.sql.Connection; import java.sql.DriverManager; import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; /** * A {@link ConnectionProvider} which requests connections directly from {@link DriverManager}. * *

Unlikely to be suitable for most production applications since it doesn't use any sort of * connection pool. * *

Usage: * * 

ConnectionProvider provider = SimpleConnectionProvider.builder()
 *   .driverClassName("org.postgresql.Driver")
 *   .url(myJdbcUrl)
 *   .user("myusername")
 *   .password("mypassword")
 *   .build()
*/ @SuperBuilder @Slf4j final class DriverConnectionProvider implements ConnectionProvider, Validatable { private final String driverClassName; private final String url; private final String user; private final String password; private volatile boolean initialized; @Override public Connection obtainConnection() { return uncheckedly( () -> { if (!initialized) { synchronized (this) { log.debug("Initialising {}", driverClassName); Class.forName(driverClassName); initialized = true; } } log.debug("Opening connection to {}", url); Connection connection = DriverManager.getConnection(url, user, password); connection.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); return connection; }); } @Override public void validate(Validator validator) { validator.notBlank("driverClassName", driverClassName); validator.notBlank("url", url); validator.notBlank("user", user); validator.notBlank("password", password); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/ExecutorSubmitter.java ================================================ package com.gruelbox.transactionoutbox; import com.gruelbox.transactionoutbox.spi.Utils; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; import java.util.function.Consumer; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.slf4j.event.Level; /** * Schedules background work using a local {@link Executor} implementation. Note that the {@link * Runnable}s submitted to this will not be {@link java.io.Serializable} so will not be suitable for * remoting. Remote submission of work is not yet supported. * *

Note that there are some important aspects that should be considered in the configuration of * this executor: * *

    *
  • Should use a BOUNDED blocking queue implementation such as {@link ArrayBlockingQueue}, * otherwise under high volume, the queue may get so large it causes out-of-memory errors. *
  • Should use a {@link java.util.concurrent.RejectedExecutionHandler} which either throws * (such as {@link java.util.concurrent.ThreadPoolExecutor.AbortPolicy}), silently fails (such * as {@link java.util.concurrent.ThreadPoolExecutor.DiscardPolicy}) or blocks the calling * thread until a thread is available. It should not execute the work in the * calling thread (e.g. {@link java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy}, * since this could result in unpredictable effects with tasks assuming they will be run in a * different thread context corrupting thread state. Generally, throwing or silently failing * are preferred since this allows the database to absorb all backpressure, but if you have a * strong reason to choose a blocking policy to enforce upstream backpressure, be aware that * {@link TransactionOutbox#flush()} can potentially block for a long period of time too, so * design any background processing which calls it accordingly (e.g. avoid calling from a * timed scheduled job; perhaps instead simply loop it). *
  • The queue can afford to be quite large in most realistic production deployments, and it is * advised that it be so (10000+). *
*/ @Slf4j @Builder public class ExecutorSubmitter implements Submitter, Validatable { /** * @param executor The executor to use. */ @SuppressWarnings("JavaDoc") private final Executor executor; /** * @param logLevelWorkQueueSaturation The log level to use when work submission hits the executor * queue limit. This usually indicates saturation and may be of greater interest than the * default {@code DEBUG} level. */ @SuppressWarnings("JavaDoc") @Builder.Default private final Level logLevelWorkQueueSaturation = Level.DEBUG; @Override public void submit(TransactionOutboxEntry entry, Consumer localExecutor) { try { executor.execute(() -> localExecutor.accept(entry)); log.debug("Submitted {} for immediate processing", entry.description()); } catch (RejectedExecutionException e) { Utils.logAtLevel( log, logLevelWorkQueueSaturation, "Queued {} for processing when executor is available", entry.description()); } catch (Exception e) { log.warn( "Failed to submit {} for execution. It will be re-attempted later.", entry.description(), e); } } @Override public void validate(Validator validator) { validator.notNull("executor", executor); validator.notNull("logLevelWorkQueueSaturation", logLevelWorkQueueSaturation); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/FailedDeserializingInvocation.java ================================================ package com.gruelbox.transactionoutbox; import java.io.IOException; import java.lang.reflect.InvocationTargetException; /** Represents an invocation those deserialization failed. */ public class FailedDeserializingInvocation extends Invocation { /** * @return Indicates an Exception during De-Serialization, that is not persisted. */ private final transient IOException exceptionDuringDeserialization; public FailedDeserializingInvocation(IOException exceptionDuringDeserialization) { super("", "", new Class[] {}, new Object[] {}); this.exceptionDuringDeserialization = exceptionDuringDeserialization; } @Override void invoke(Object instance, TransactionOutboxListener listener) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { throw new UncheckedException(exceptionDuringDeserialization); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/FunctionInstantiator.java ================================================ package com.gruelbox.transactionoutbox; import com.gruelbox.transactionoutbox.spi.AbstractFullyQualifiedNameInstantiator; import java.util.function.Function; import lombok.experimental.SuperBuilder; @SuperBuilder class FunctionInstantiator extends AbstractFullyQualifiedNameInstantiator { private final Function, Object> fn; @Override public Object createInstance(Class clazz) { return fn.apply(clazz); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/Instantiator.java ================================================ package com.gruelbox.transactionoutbox; import java.util.function.Function; /** * Provides callbacks for the creation and serialization of classes by {@link TransactionOutbox}. */ public interface Instantiator { /** * Creates an {@link Instantiator} which records the class name as its fully qualified name (e.g. * {@code com.gruelbox.example.EnterpriseBeanProxyFactoryFactory}) and instantiates instances * using reflection and a no-args constructor. * *

This is the default used by {@link TransactionOutbox} if nothing else is specified. * * @return A reflection instantiator */ static Instantiator usingReflection() { return ReflectionInstantiator.builder().build(); } /** * Creates an {@link Instantiator} which records the class name as its fully qualified name (e.g. * {@code com.gruelbox.example.EnterpriseBeanProxyFactoryFactory}) and instantiates instances * using the supplied function, which takes the fully qualified name and should return an * instance. * *

This is a good option to use with dependency injection frameworks such as Guice: * * 

TransactionOutbox outbox = TransactionOutbox.builder()
   * ...
   * .instantiator(Instantiator.using(injector::getInstance))
   * .build();
* * @param fn A function to create an instance of the specified class. * @return A reflection instantiator */ static Instantiator using(Function, Object> fn) { return FunctionInstantiator.builder().fn(fn).build(); } /** * Provides the name of the specified class. This may be the classes fully-qualified name, or may * be an alias of some kind. This is up to the implementer. * *

Not using the actual class name can be useful in avoiding a case where queued tasks end up * referencing renamed classes following a refactor. It is also useful for DI frameworks such as * Spring DI, which use named bindings by default. * * @param clazz The class to get the name of. * @return The class name. */ String getName(Class clazz); /** * Requests an instance of the named class, where the "name" is whatever is returned by {@link * #getName(Class)}. * *

A common use-case for this method is to return a class from a DI framework such as Guice * (using an injected {code Injector}), but it is perfectly valid to simply instantiate the class * by name and populate its dependencies directly. * * @param name The class "name" as returned by {@link #getName(Class)}. * @return An instance of the class. */ Object getInstance(String name); } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/Invocation.java ================================================ package com.gruelbox.transactionoutbox; import com.google.gson.annotations.SerializedName; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Callable; import lombok.*; import lombok.experimental.FieldDefaults; import lombok.extern.slf4j.Slf4j; import org.slf4j.MDC; /** * Represents the invocation of a specific method on a named class (where the name is provided by an * {@link Instantiator}), with the specified arguments. * *

Optimized for safe serialization via GSON. */ @SuppressWarnings("WeakerAccess") @ToString @EqualsAndHashCode @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) @Getter @Slf4j public class Invocation { /** * @return The class name (as provided/expected by an {@link Instantiator}). */ @SuppressWarnings("JavaDoc") @SerializedName("c") String className; /** * @return The method name. Combined with {@link #parameterTypes}, uniquely identifies the method. */ @SuppressWarnings("JavaDoc") @SerializedName("m") String methodName; /** * @return The method parameter types. Combined with {@link #methodName}, uniquely identifies the * method. */ @SuppressWarnings("JavaDoc") @SerializedName("p") Class[] parameterTypes; /** * @return The arguments to call. Must match {@link #parameterTypes}. */ @SuppressWarnings("JavaDoc") @SerializedName("a") Object[] args; /** * @return Thread-local context to recreate when running the task. */ @SuppressWarnings("JavaDoc") @SerializedName("x") Map mdc; /** * @return Free-form data used by add-ons to store additional information related to the request */ @SerializedName("s") Map session; /** * @param className The class name (as provided/expected by an {@link Instantiator}). * @param methodName The method name. Combined with {@link #parameterTypes}, uniquely identifies * the method. * @param parameterTypes The method parameter types. Combined with {@link #methodName}, uniquely * identifies the method. * @param args The arguments to call. Must match {@link #parameterTypes}. */ public Invocation(String className, String methodName, Class[] parameterTypes, Object[] args) { this(className, methodName, parameterTypes, args, null, null); } /** * @param className The class name (as provided/expected by an {@link Instantiator}). * @param methodName The method name. Combined with {@link #parameterTypes}, uniquely identifies * the method. * @param parameterTypes The method parameter types. Combined with {@link #methodName}, uniquely * identifies the method. * @param args The arguments to call. Must match {@link #parameterTypes}. * @param mdc Thread-local context to recreate when running the task. * @deprecated Use another constructor. */ @Deprecated(forRemoval = true) public Invocation( String className, String methodName, Class[] parameterTypes, Object[] args, Map mdc) { this(className, methodName, parameterTypes, args, mdc, null); } /** * @param className The class name (as provided/expected by an {@link Instantiator}). * @param methodName The method name. Combined with {@link #parameterTypes}, uniquely identifies * the method. * @param parameterTypes The method parameter types. Combined with {@link #methodName}, uniquely * identifies the method. * @param args The arguments to call. Must match {@link #parameterTypes}. * @param mdc Thread-local context to recreate when running the task. * @param session Free-form data used by add-ons to store additional information related to the * request. */ public Invocation( String className, String methodName, Class[] parameterTypes, Object[] args, Map mdc, Map session) { this.className = className; this.methodName = methodName; this.parameterTypes = parameterTypes; this.args = args; this.mdc = mdc == null ? null : new HashMap<>(mdc); this.session = session == null ? null : new HashMap<>(session); } T withinMDC(Callable callable) throws Exception { if (mdc != null && MDC.getMDCAdapter() != null) { var oldMdc = MDC.getCopyOfContextMap(); MDC.setContextMap(mdc); try { return callable.call(); } finally { if (oldMdc == null) { MDC.clear(); } else { MDC.setContextMap(oldMdc); } } } else { return callable.call(); } } void invoke(Object instance, TransactionOutboxListener listener) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { Method method = instance.getClass().getDeclaredMethod(methodName, parameterTypes); method.setAccessible(true); if (log.isDebugEnabled()) { log.debug("Invoking method {} with args {}", method, Arrays.toString(args)); } listener.wrapInvocation( new TransactionOutboxListener.Invocator() { @Override public void run() throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { method.invoke(instance, args); } @Override public Invocation getInvocation() { return Invocation.this; } }); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/InvocationSerializer.java ================================================ package com.gruelbox.transactionoutbox; import java.io.IOException; import java.io.Reader; import java.io.Writer; /** * {@link Invocation} objects are inherently difficult to serialize safely since they are * unpredictably polymorphic. Allowing them to contain any type reference opens you up to a * host of code injection attacks. At the same time, allowing possibly-unstable types into * serialized {@link Invocation}s can result in compatibility issues, with still unprocessed entries * in the database containing older versions of your classes. To avoid this, it makes sense to * specify the types supported and restrict this set of serializable classes to known-stable types * such as primitives and common JDK value types. {@link #createDefaultJsonSerializer()} provides * exactly this and is used by default. However, if you want to extend this list or use a different * serialization format, you can create your own implementation here, at your own risk. */ public interface InvocationSerializer { /** * Creates a locked-down serializer which supports a limited list of primitives and simple JDK * value types. Shortcut to {@link DefaultInvocationSerializer}. * * @return The serializer. */ static InvocationSerializer createDefaultJsonSerializer() { return DefaultInvocationSerializer.builder().build(); } /** * Serializes an invocation to the supplied writer. * * @param invocation The invocation. * @param writer The writer. */ void serializeInvocation(Invocation invocation, Writer writer); /** * Deserializes an invocation from the supplied reader. * * @param reader The reader. * @return The deserialized invocation. */ Invocation deserializeInvocation(Reader reader) throws IOException; } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/Migration.java ================================================ package com.gruelbox.transactionoutbox; import lombok.Value; /** A database migration script entry. See {@link Dialect#getMigrations()}. */ @Value public class Migration { int version; String name; String sql; public Migration withSql(String sql) { return new Migration(version, name, sql); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/MissingOptionalDependencyException.java ================================================ package com.gruelbox.transactionoutbox; public class MissingOptionalDependencyException extends RuntimeException { public MissingOptionalDependencyException(String groupId, String artifactId) { super( String.format( "You are trying to use an optional feature, which requires an additional dependency (%s:%s). Please add it to your classpath, and try again.", groupId, artifactId)); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/NoTransactionActiveException.java ================================================ package com.gruelbox.transactionoutbox; /** Thrown if an active transaction is required by a method and no transaction is active. */ @SuppressWarnings("WeakerAccess") public final class NoTransactionActiveException extends RuntimeException { public NoTransactionActiveException() { super(); } public NoTransactionActiveException(Throwable cause) { super(cause); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/OptimisticLockException.java ================================================ package com.gruelbox.transactionoutbox; /** Thrown when we attempt to update a record which has been modified by another thread. */ public class OptimisticLockException extends Exception {} ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/ParameterContextTransactionManager.java ================================================ package com.gruelbox.transactionoutbox; import java.lang.reflect.Method; import java.util.Arrays; /** * A transaction manager which makes no assumption of a "current" {@link Transaction}. This means * that {@link TransactionOutbox#schedule(Class)} needs to be given the transaction to use as part * of any invoked method's arguments. In turn, that method will need the transaction at the time it * is invoked. * *

Call patterns permitted: * * 

 * // Using TransactionManager
 * transactionManager.inTransaction(tx ->
 *   outbox.schedule(MyClass.class).myMethod("foo", tx));
 *
 * // Using some third party transaction manager
 * wibbleTransactionManager.doInATransaction(context ->
 *   outbox.schedule(MyClass.class).myMethod("foo", context));
 *
*/ public interface ParameterContextTransactionManager extends TransactionManager { /** * Given an implementation-specific transaction context, return the active {@link Transaction}. * * @param context The implementation-specific context, of the same type returned by {@link * #contextType()}. * @return The transaction, or null if the context is not known. */ Transaction transactionFromContext(T context); /** * @return The type expected by {@link #transactionFromContext(Object)}. */ Class contextType(); /** * Obtains the active transaction by parsing the method arguments for a {@link Transaction} or a * context (any object of type {@link #contextType()}). All such arguments are removed from the * invocation adn replaced with nulls before saving. They will be "rehydrated" later upon actual * invocation using the transaction/context at the time of invocation. * * @param method The method called. * @param args The method arguments. * @return The transactional invocation. */ @SuppressWarnings("unchecked") @Override default TransactionalInvocation extractTransaction(Method method, Object[] args) { args = Arrays.copyOf(args, args.length); var params = Arrays.copyOf(method.getParameterTypes(), method.getParameterCount()); Transaction transaction = null; for (int i = 0; i < args.length; i++) { Object candidate = args[i]; if (candidate instanceof Transaction) { transaction = (Transaction) candidate; args[i] = null; } else if (contextType().isInstance(candidate)) { if (transaction == null) { transaction = transactionFromContext((T) candidate); if (transaction == null) { throw new IllegalArgumentException( candidate.getClass().getName() + " context passed to " + method + " does not relate to a known transaction. This either indicates that the context object was not " + "created by normal means or the transaction manager is incorrectly configured."); } } args[i] = null; params[i] = TransactionContextPlaceholder.class; } } if (transaction == null) { throw new IllegalArgumentException( getClass().getName() + " requires transaction context (either " + contextType().getName() + " or " + Transaction.class.getName() + ") to be passed as a parameter to any scheduled method."); } return new TransactionalInvocation( method.getDeclaringClass(), method.getName(), params, args, transaction); } /** * Modifies an {@link Invocation} at runtime to rehyrate it with the transaction context in which * the record was locked. * * @param invocation The invocation. * @param transaction The transaction to use. * @return The modified invocation. */ @Override default Invocation injectTransaction(Invocation invocation, Transaction transaction) { Object[] args = Arrays.copyOf(invocation.getArgs(), invocation.getArgs().length); Class[] params = Arrays.copyOf(invocation.getParameterTypes(), invocation.getParameterTypes().length); for (int i = 0; i < invocation.getParameterTypes().length; i++) { Class parameterType = invocation.getParameterTypes()[i]; if (Transaction.class.isAssignableFrom(parameterType)) { if (args[i] != null) { throw new IllegalArgumentException( String.format( "Parameter %s.%s[%d] contains unexpected serialized Transaction", invocation.getClassName(), invocation.getMethodName(), i)); } args[i] = transaction; } else if (parameterType.equals(TransactionContextPlaceholder.class)) { if (args[i] != null) { throw new IllegalArgumentException( String.format( "Parameter %s.%s[%d] contains unexpected serialized Transaction context", invocation.getClassName(), invocation.getMethodName(), i)); } args[i] = transaction.context(); params[i] = contextType(); } } return new Invocation( invocation.getClassName(), invocation.getMethodName(), params, args, invocation.getMdc(), invocation.getSession()); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/Persistor.java ================================================ package com.gruelbox.transactionoutbox; import java.time.Instant; import java.util.Collection; import java.util.List; /** * Saves and loads {@link TransactionOutboxEntry}s. For most use cases, just use {@link * DefaultPersistor}. It is parameterisable and designed for extension, so can be easily modified. * Creating completely new implementations of {@link Persistor} should be reserved for cases where * the underlying data store is of a completely different nature entirely. */ public interface Persistor { /** * Uses the default relational persistor. Shortcut for: * DefaultPersistor.builder().dialect(dialect).build(); * * @param dialect The database dialect. * @return The persistor. */ static DefaultPersistor forDialect(Dialect dialect) { return DefaultPersistor.builder().dialect(dialect).build(); } /** * Upgrades any database schema used by the persistor to the latest version. Called on creation of * a {@link TransactionOutbox}. * * @param transactionManager The transactoin manager. */ void migrate(TransactionManager transactionManager); /** * Saves a new {@link TransactionOutboxEntry}. Should throw {@link AlreadyScheduledException} if * the record already exists based on the {@code id} or {@code uniqueRequestId} (the latter of * which should not treat nulls as duplicates). * * @param tx The current {@link Transaction}. * @param entry The entry to save. All properties on the object should be saved recursively. * @throws Exception Any exception. */ void save(Transaction tx, TransactionOutboxEntry entry) throws Exception; /** * Used in tests to simulate a database reload. * * @param invocation An invocation. * @return The same invocation passed through a serialize/deserialize loop. */ default Invocation serializeAndDeserialize(Invocation invocation) { return invocation; } /** * Deletes a {@link TransactionOutboxEntry}. * *

A record should only be deleted if both the {@code id} and {@code version} on the * database match that on the object. If no such record is found, {@link OptimisticLockException} * should be thrown. * * @param tx The current {@link Transaction}. * @param entry The entry to be deleted. * @throws OptimisticLockException If no such record is found. * @throws Exception Any other exception. */ void delete(Transaction tx, TransactionOutboxEntry entry) throws Exception; /** * Modifies an existing {@link TransactionOutboxEntry}. Performs an optimistic lock check on any * existing record via a compare-and-swap operation and throws {@link OptimisticLockException} if * the lock is failed. {@link TransactionOutboxEntry#setVersion(int)} is called before returning * containing the new version of the entry. * * @param tx The current {@link Transaction}. * @param entry The entry to be updated. * @throws OptimisticLockException If no record with same id and version is found. * @throws Exception Any other exception. */ void update(Transaction tx, TransactionOutboxEntry entry) throws Exception; /** * Attempts to pessimistically lock an existing {@link TransactionOutboxEntry}. * * @param tx The current {@link Transaction}. * @param entry The entry to be locked * @return true if the lock was successful. * @throws OptimisticLockException If no record with same id and version is found. * @throws Exception Any other exception. */ boolean lock(Transaction tx, TransactionOutboxEntry entry) throws Exception; /** * Clears the blocked flag and resets the attempt count to zero. * * @param tx The current {@link Transaction}. * @param entryId The entry id. * @return true if the update was successful. This will be false if the record was no longer * blocked or didn't exist anymore. * @throws Exception Any other exception. */ boolean unblock(Transaction tx, String entryId) throws Exception; /** * Selects up to a specified maximum number of non-blocked records which have passed their {@link * TransactionOutboxEntry#getNextAttemptTime()}. Until a subsequent call to {@link * #lock(Transaction, TransactionOutboxEntry)}, these records may be selected by another instance * for processing. * * @param tx The current {@link Transaction}. * @param batchSize The number of records to select. * @param now The time to use when selecting records. * @return The records. * @throws Exception Any exception. */ List selectBatch(Transaction tx, int batchSize, Instant now) throws Exception; /** * Selects the next items in all the open topics as a batch for processing. Does not lock. * * @param tx The current {@link Transaction}. * @param batchSize The maximum number of records to select. * @param now The time to use when selecting records. * @return The records. * @throws Exception Any exception. */ Collection selectNextInTopics(Transaction tx, int batchSize, Instant now) throws Exception; /** * Selects the next items in all selected topics as a batch for processing. Does not lock. * * @param tx The current {@link Transaction}. * @param topicNames The topics to select records from. * @param batchSize The maximum number of records to select. * @param now The time to use when selecting records. * @return The records. * @throws Exception Any exception. */ Collection selectNextInSelectedTopics( Transaction tx, List topicNames, int batchSize, Instant now) throws Exception; /** * Deletes records which have processed and passed their expiry time, in specified batch sizes. * * @param tx The current {@link Transaction}. * @param batchSize The maximum number of records to select. * @param now The time to use when selecting records. * @return The number of records deleted. * @throws Exception Any exception. */ int deleteProcessedAndExpired(Transaction tx, int batchSize, Instant now) throws Exception; /** * Checks the connection status of a transaction. * * @param tx The current {@link Transaction}. * @return true if connected and working. */ boolean checkConnection(Transaction tx) throws Exception; /** * Clears the database. For testing only. * * @param tx The current {@link Transaction}. */ void clear(Transaction tx) throws Exception; } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/ReflectionInstantiator.java ================================================ package com.gruelbox.transactionoutbox; import com.gruelbox.transactionoutbox.spi.AbstractFullyQualifiedNameInstantiator; import com.gruelbox.transactionoutbox.spi.Utils; import java.lang.reflect.Constructor; import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; /** * {@link Instantiator} which records the class name as its fully-qualified class name, and * instantiates via reflection. The class must have a no-args constructor. Likely only of use in * simple applications since it does not allow for dependency injection. */ @Slf4j @SuperBuilder final class ReflectionInstantiator extends AbstractFullyQualifiedNameInstantiator { @Override public Object createInstance(Class clazz) { log.debug("Getting instance of class [{}] via reflection", clazz.getName()); Constructor constructor = Utils.uncheckedly(clazz::getDeclaredConstructor); constructor.setAccessible(true); return Utils.uncheckedly(constructor::newInstance); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/RuntimeTypeAdapterFactory.java ================================================ /* * Copyright (C) 2011 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.gruelbox.transactionoutbox; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; import com.google.gson.JsonPrimitive; import com.google.gson.TypeAdapter; import com.google.gson.TypeAdapterFactory; import com.google.gson.internal.Streams; import com.google.gson.reflect.TypeToken; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonWriter; import java.io.IOException; import java.util.LinkedHashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; /** * Adapts values whose runtime type may differ from their declaration type. This is necessary when a * field's type is not the same type that GSON should create when deserializing that field. For * example, consider these types: * * 

{@code
 * abstract class Shape {
 *   int x;
 *   int y;
 * }
 * class Circle extends Shape {
 *   int radius;
 * }
 * class Rectangle extends Shape {
 *   int width;
 *   int height;
 * }
 * class Diamond extends Shape {
 *   int width;
 *   int height;
 * }
 * class Drawing {
 *   Shape bottomShape;
 *   Shape topShape;
 * }
 * }
* *

Without additional type information, the serialized JSON is ambiguous. Is the bottom shape in * this drawing a rectangle or a diamond? * * 

{@code
 * {
 *   "bottomShape": {
 *     "width": 10,
 *     "height": 5,
 *     "x": 0,
 *     "y": 0
 *   },
 *   "topShape": {
 *     "radius": 2,
 *     "x": 4,
 *     "y": 1
 *   }
 * }
 * }
* * This class addresses this problem by adding type information to the serialized JSON and honoring * that type information when the JSON is deserialized: * * 
{@code
 * {
 *   "bottomShape": {
 *     "type": "Diamond",
 *     "width": 10,
 *     "height": 5,
 *     "x": 0,
 *     "y": 0
 *   },
 *   "topShape": {
 *     "type": "Circle",
 *     "radius": 2,
 *     "x": 4,
 *     "y": 1
 *   }
 * }
 * }
* * Both the type field name ({@code "type"}) and the type labels ({@code "Rectangle"}) are * configurable. * *

Registering Types

* * Create a {@code RuntimeTypeAdapterFactory} by passing the base type and type field name to the * {@link #of} factory method. If you don't supply an explicit type field name, {@code "type"} will * be used. * * 
{@code
 * RuntimeTypeAdapterFactory shapeAdapterFactory
 *     = RuntimeTypeAdapterFactory.of(Shape.class, "type");
 * }
* * Next register all of your subtypes. Every subtype must be explicitly registered. This protects * your application from injection attacks. If you don't supply an explicit type label, the type's * simple name will be used. * * 
{@code
 * shapeAdapterFactory.registerSubtype(Rectangle.class, "Rectangle");
 * shapeAdapterFactory.registerSubtype(Circle.class, "Circle");
 * shapeAdapterFactory.registerSubtype(Diamond.class, "Diamond");
 * }
* * Finally, register the type adapter factory in your application's GSON builder: * * 
{@code
 * Gson gson = new GsonBuilder()
 *     .registerTypeAdapterFactory(shapeAdapterFactory)
 *     .create();
 * }
* * Like {@code GsonBuilder}, this API supports chaining: * * 
{@code
 * RuntimeTypeAdapterFactory shapeAdapterFactory = RuntimeTypeAdapterFactory.of(Shape.class)
 *     .registerSubtype(Rectangle.class)
 *     .registerSubtype(Circle.class)
 *     .registerSubtype(Diamond.class);
 * }
* *

Serialization and deserialization

* * In order to serialize and deserialize a polymorphic object, you must specify the base type * explicitly. * * 
{@code
 * Diamond diamond = new Diamond();
 * String json = gson.toJson(diamond, Shape.class);
 * }
* * And then: * * 
{@code
 * Shape shape = gson.fromJson(json, Shape.class);
 * }
*/ @Slf4j final class RuntimeTypeAdapterFactory implements TypeAdapterFactory { private final Class baseType; private final String typeFieldName; private final Map> labelToSubtype = new LinkedHashMap<>(); private final Map, String> subtypeToLabel = new LinkedHashMap<>(); private final boolean maintainType; private RuntimeTypeAdapterFactory(Class baseType, String typeFieldName, boolean maintainType) { if (typeFieldName == null || baseType == null) { throw new NullPointerException(); } this.baseType = baseType; this.typeFieldName = typeFieldName; this.maintainType = maintainType; } /** * Creates a new runtime type adapter for {@code baseType} using {@code "type"} as the type field * name. */ static RuntimeTypeAdapterFactory of(Class baseType) { return new RuntimeTypeAdapterFactory<>(baseType, "type", false); } /** * Registers {@code type} identified by {@code label}. Labels are case sensitive. * * @throws IllegalArgumentException if either {@code type} or {@code label} have already been * registered on this type adapter. */ RuntimeTypeAdapterFactory registerSubtype(Class type, String label) { if (type == null || label == null) { throw new NullPointerException(); } if (subtypeToLabel.containsKey(type) || labelToSubtype.containsKey(label)) { throw new IllegalArgumentException("types and labels must be unique"); } labelToSubtype.put(label, type); subtypeToLabel.put(type, label); return this; } @Override public TypeAdapter create(Gson gson, TypeToken type) { if (type.getRawType() != baseType) { return null; } log.debug("Looking for adapter for {}", type); final Map> labelToDelegate = new LinkedHashMap<>(); final Map, TypeAdapter> subtypeToDelegate = new LinkedHashMap<>(); for (Map.Entry> entry : labelToSubtype.entrySet()) { TypeAdapter delegate = gson.getDelegateAdapter(this, TypeToken.get(entry.getValue())); labelToDelegate.put(entry.getKey(), delegate); subtypeToDelegate.put(entry.getValue(), delegate); } return new TypeAdapter() { @Override public R read(JsonReader in) { log.debug("Reading"); JsonElement jsonElement = Streams.parse(in); JsonElement labelJsonElement; if (maintainType) { labelJsonElement = jsonElement.getAsJsonObject().get(typeFieldName); } else { labelJsonElement = jsonElement.getAsJsonObject().remove(typeFieldName); } if (labelJsonElement == null) { throw new JsonParseException( "cannot deserialize " + baseType + " because it does not define a field named " + typeFieldName); } String label = labelJsonElement.getAsString(); @SuppressWarnings("unchecked") // registration requires that subtype extends T TypeAdapter delegate = (TypeAdapter) labelToDelegate.get(label); if (delegate == null) { throw new JsonParseException( "cannot deserialize " + baseType + " subtype named " + label + "; did you forget to register a subtype?"); } return delegate.fromJsonTree(jsonElement); } @Override public void write(JsonWriter out, R value) throws IOException { Class srcType = value.getClass(); String label = subtypeToLabel.get(srcType); @SuppressWarnings("unchecked") // registration requires that subtype extends T TypeAdapter delegate = (TypeAdapter) subtypeToDelegate.get(srcType); if (delegate == null) { throw new JsonParseException( "cannot serialize " + srcType.getName() + "; did you forget to register a subtype?"); } JsonObject jsonObject = delegate.toJsonTree(value).getAsJsonObject(); if (maintainType) { Streams.write(jsonObject, out); return; } JsonObject clone = new JsonObject(); if (jsonObject.has(typeFieldName)) { throw new JsonParseException( "cannot serialize " + srcType.getName() + " because it already defines a field named " + typeFieldName); } clone.add(typeFieldName, new JsonPrimitive(label)); for (Map.Entry e : jsonObject.entrySet()) { clone.add(e.getKey(), e.getValue()); } Streams.write(clone, out); } }.nullSafe(); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/SQLAction.java ================================================ package com.gruelbox.transactionoutbox; import java.sql.Connection; import java.sql.SQLException; @FunctionalInterface interface SQLAction { void doAction(Connection connection) throws SQLException; } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/SimpleTransactionManager.java ================================================ package com.gruelbox.transactionoutbox; import com.gruelbox.transactionoutbox.spi.AbstractThreadLocalTransactionManager; import com.gruelbox.transactionoutbox.spi.SimpleTransaction; import com.gruelbox.transactionoutbox.spi.Utils; import java.sql.Connection; import java.sql.SQLException; import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; /** * A simple {@link TransactionManager} implementation suitable for applications with no existing * transaction management. */ @SuperBuilder @Slf4j final class SimpleTransactionManager extends AbstractThreadLocalTransactionManager { private final ConnectionProvider connectionProvider; @Override public T inTransactionReturnsThrows( ThrowingTransactionalSupplier work) throws E { return withTransaction( atx -> { T result = processAndCommitOrRollback(work, (SimpleTransaction) atx); ((SimpleTransaction) atx).processHooks(); return result; }); } private T processAndCommitOrRollback( ThrowingTransactionalSupplier work, SimpleTransaction transaction) throws E { try { log.debug("Processing work"); T result = work.doWork(transaction); transaction.flushBatches(); log.debug("Committing transaction"); transaction.commit(); return result; } catch (Exception e) { try { log.warn( "Exception in transactional block ({}{}). Rolling back. See later messages for detail", e.getClass().getSimpleName(), e.getMessage() == null ? "" : (" - " + e.getMessage())); transaction.rollback(); } catch (Exception ex) { log.warn("Failed to roll back", ex); } throw e; } } private T withTransaction(ThrowingTransactionalSupplier work) throws E { try (Connection connection = connectionProvider.obtainConnection(); SimpleTransaction transaction = pushTransaction(new SimpleTransaction(connection, null))) { log.debug("Got connection {}", connection); boolean autoCommit = transaction.connection().getAutoCommit(); if (autoCommit) { log.debug("Setting auto-commit false"); Utils.uncheck(() -> transaction.connection().setAutoCommit(false)); } try { return work.doWork(transaction); } finally { connection.setAutoCommit(autoCommit); } } catch (SQLException e) { throw new RuntimeException(e); } finally { popTransaction(); } } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/StubParameterContextTransactionManager.java ================================================ package com.gruelbox.transactionoutbox; import com.gruelbox.transactionoutbox.spi.ProxyFactory; import com.gruelbox.transactionoutbox.spi.SimpleTransaction; import com.gruelbox.transactionoutbox.spi.Utils; import java.sql.Connection; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; /** * A stub transaction manager that assumes no underlying database, and a transaction context of the * specified type. */ @Slf4j public class StubParameterContextTransactionManager implements ParameterContextTransactionManager { private final Class contextClass; private final Supplier contextFactory; private final ConcurrentMap contextMap = new ConcurrentHashMap<>(); /** * @param contextClass The class that represents the context. Must support equals/hashCode. * @param contextFactory Generates context instances when transactions are started. */ public StubParameterContextTransactionManager(Class contextClass, Supplier contextFactory) { this.contextClass = contextClass; this.contextFactory = contextFactory; } @Override public Transaction transactionFromContext(C context) { return contextMap.get(context); } @Override public final Class contextType() { return contextClass; } @Override public T inTransactionReturnsThrows( ThrowingTransactionalSupplier work) throws E { return withTransaction( atx -> { T result = work.doWork(atx); ((SimpleTransaction) atx).processHooks(); return result; }); } private T withTransaction(ThrowingTransactionalSupplier work) throws E { Connection mockConnection = Utils.createLoggingProxy(new ProxyFactory(), Connection.class); C context = contextFactory.get(); try (SimpleTransaction transaction = new SimpleTransaction(mockConnection, context)) { contextMap.put(context, transaction); return work.doWork(transaction); } finally { contextMap.remove(context); } } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/StubPersistor.java ================================================ package com.gruelbox.transactionoutbox; import java.time.Instant; import java.util.Collection; import java.util.List; import lombok.Builder; /** Stub implementation of {@link Persistor}. */ @Builder public class StubPersistor implements Persistor { StubPersistor() {} @Override public void migrate(TransactionManager transactionManager) { // No-op } @Override public void save(Transaction tx, TransactionOutboxEntry entry) { // No-op } @Override public void delete(Transaction tx, TransactionOutboxEntry entry) { // No-op } @Override public void update(Transaction tx, TransactionOutboxEntry entry) { // No-op } @Override public boolean lock(Transaction tx, TransactionOutboxEntry entry) { return true; } @Override public boolean unblock(Transaction tx, String entryId) { return true; } @Override public List selectBatch(Transaction tx, int batchSize, Instant now) { return List.of(); } @Override public Collection selectNextInTopics( Transaction tx, int flushBatchSize, Instant now) { return List.of(); } @Override public Collection selectNextInSelectedTopics( Transaction tx, List topicNames, int batchSize, Instant now) throws Exception { return List.of(); } @Override public int deleteProcessedAndExpired(Transaction tx, int batchSize, Instant now) { return 0; } @Override public void clear(Transaction tx) {} @Override public boolean checkConnection(Transaction tx) { return true; } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/StubThreadLocalTransactionManager.java ================================================ package com.gruelbox.transactionoutbox; import com.gruelbox.transactionoutbox.spi.AbstractThreadLocalTransactionManager; import com.gruelbox.transactionoutbox.spi.ProxyFactory; import com.gruelbox.transactionoutbox.spi.SimpleTransaction; import com.gruelbox.transactionoutbox.spi.Utils; import java.sql.Connection; import lombok.extern.slf4j.Slf4j; /** * A stub transaction manager that assumes no underlying database and thread local transaction * management. */ @Slf4j public class StubThreadLocalTransactionManager extends AbstractThreadLocalTransactionManager { public StubThreadLocalTransactionManager() { // Nothing to do } @Override public T inTransactionReturnsThrows( ThrowingTransactionalSupplier work) throws E { return withTransaction( atx -> { T result = work.doWork(atx); ((SimpleTransaction) atx).processHooks(); return result; }); } private T withTransaction(ThrowingTransactionalSupplier work) throws E { Connection mockConnection = Utils.createLoggingProxy(new ProxyFactory(), Connection.class); try (SimpleTransaction transaction = pushTransaction(new SimpleTransaction(mockConnection, null))) { return work.doWork(transaction); } finally { popTransaction(); } } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/Submitter.java ================================================ package com.gruelbox.transactionoutbox; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.Executor; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; /** Called by {@link TransactionOutbox} to submit work for background processing. */ public interface Submitter { /** * Schedules background work using a local {@link Executor} implementation. * *

Shortcut for {@code ExecutorSubmitter.builder().executor(executor).build()}. * * @param executor The executor. * @return The submitter. */ static Submitter withExecutor(Executor executor) { return ExecutorSubmitter.builder().executor(executor).build(); } /** * Schedules background worh with a {@link ThreadPoolExecutor}, sized to match {@link * ForkJoinPool#commonPool()} (or one thread, whichever is the larger), with a maximum queue size * of 16384 before work is discarded. * * @return The submitter. */ static Submitter withDefaultExecutor() { // JDK bug means this warning can't be fixed //noinspection Convert2Diamond return withExecutor( new ThreadPoolExecutor( 1, Math.max(1, ForkJoinPool.commonPool().getParallelism()), 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue(16384))); } /** * Submits a transaction outbox task for processing. The {@link TransactionOutboxEntry} is * provided, along with a {@code localExecutor} which can run the work immediately. An * implementation may validly do any of the following: * *

    *
  • Submit a call to {@code localExecutor} in a local thread, e.g. using an {@link Executor}. * This is what implementations returned by {@link #withExecutor(Executor)} or {@link * #withDefaultExecutor()} will do, and is recommended in almost all cases. *
  • Serialize the {@link TransactionOutboxEntry}, send it to another instance (e.g. via a * queue) and have the handler code call {@link * TransactionOutbox#processNow(TransactionOutboxEntry)}. Such an approach should not * generally be necessary since {@link TransactionOutbox#flush()} is designed to be called * repeatedly on multiple instances. This means there is a degree of load balancing built * into the system, but when dealing with very high load, very low run-time tasks, this can * get overwhelmed and direct multi-instance queuing can help balance the load at source. * Note: it is recommended that the {@code invocation} property of the * {@link TransactionOutboxEntry} be serialized using {@link * InvocationSerializer#createDefaultJsonSerializer()} *
  • Pass the {@code entry} directly to the {@code localExecutor}. This will run the work * immediately in the calling thread and is therefore generally not recommended; the calling * thread will be either the thread calling {@link TransactionOutbox#schedule(Class)} * (effectively making the work synchronous) or the background poll thread (limiting work in * progress to one). It can, however, be useful for test cases. *
* * @param entry The entry to process. * @param localExecutor Provides a means of running the work directly locally (it is effectively * just a call to {@link TransactionOutbox#processNow(TransactionOutboxEntry)}). */ void submit(TransactionOutboxEntry entry, Consumer localExecutor); } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/ThreadLocalContextTransactionManager.java ================================================ package com.gruelbox.transactionoutbox; import java.lang.reflect.Method; /** * A transaction manager which assumes there is a single "current" {@link Transaction} on a thread * (presumably saved in a {@link ThreadLocal}) which can be both used by {@link * TransactionOutbox#schedule(Class)} as the current context to write records using {@link * Persistor} and used by scheduled methods themselves to write changes within the * transaction started as a result of reading and locking the request. * *

Call pattern permitted: * * 

transactionManager.inTransaction(() -> outbox.schedule(MyClass.ckass).myMethod("foo");
 *
* *

Adds the {@link #requireTransactionReturns(ThrowingTransactionalSupplier)} and {@link * #requireTransaction(ThrowingTransactionalWork)} methods, which extract the current transaction * from the thread context and pass it on, throwing {@link NoTransactionActiveException} if there is * no current transaction. */ public interface ThreadLocalContextTransactionManager extends TransactionManager { /** * Runs the specified work in the context of the "current" transaction (the definition of which is * up to the implementation). * * @param work Code which must be called while the transaction is active. * @param The exception type. * @throws E If any exception is thrown by {@link Runnable}. * @throws NoTransactionActiveException If a transaction is not currently active. */ default void requireTransaction(ThrowingTransactionalWork work) throws E, NoTransactionActiveException { requireTransactionReturns(ThrowingTransactionalSupplier.fromWork(work)); } /** * Runs the specified work in the context of the "current" transaction (the definition of which is * up to the implementation). * * @param work Code which must be called while the transaction is active. * @param The type returned. * @param The exception type. * @return The value returned by {@code work}. * @throws E If any exception is thrown by {@link Runnable}. * @throws NoTransactionActiveException If a transaction is not currently active. * @throws UnsupportedOperationException If the transaction manager does not support thread-local * context. */ T requireTransactionReturns(ThrowingTransactionalSupplier work) throws E, NoTransactionActiveException; /** * Obtains the active transaction by using {@link * #requireTransactionReturns(ThrowingTransactionalSupplier)}, thus requiring nothing to be passed * in the method invocation. No changes are made to the invocation. * * @param method The method called. * @param args The method arguments. * @return The transactional invocation. */ @Override default TransactionalInvocation extractTransaction(Method method, Object[] args) { return requireTransactionReturns( transaction -> new TransactionalInvocation( method.getDeclaringClass(), method.getName(), method.getParameterTypes(), args, transaction)); } /** * The transaction is not needed as part of an invocation, so the invocation is left unmodified. * * @param invocation The invocation. * @return The unmodified invocation. */ @Override default Invocation injectTransaction(Invocation invocation, Transaction transaction) { return invocation; } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/ThrowingRunnable.java ================================================ package com.gruelbox.transactionoutbox; /** A runnable... that throws. */ @FunctionalInterface public interface ThrowingRunnable { void run() throws Exception; } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/ThrowingTransactionalSupplier.java ================================================ package com.gruelbox.transactionoutbox; @FunctionalInterface public interface ThrowingTransactionalSupplier { static ThrowingTransactionalSupplier fromRunnable( Runnable runnable) { return transaction -> { runnable.run(); return null; }; } static ThrowingTransactionalSupplier fromWork( ThrowingTransactionalWork work) { return transaction -> { work.doWork(transaction); return null; }; } static ThrowingTransactionalSupplier fromWork(TransactionalWork work) { return transaction -> { work.doWork(transaction); return null; }; } static ThrowingTransactionalSupplier fromSupplier( TransactionalSupplier work) { return work::doWork; } T doWork(Transaction transaction) throws E; } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/ThrowingTransactionalWork.java ================================================ package com.gruelbox.transactionoutbox; @FunctionalInterface public interface ThrowingTransactionalWork { void doWork(Transaction transaction) throws E; } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/Transaction.java ================================================ package com.gruelbox.transactionoutbox; import java.sql.Connection; import java.sql.PreparedStatement; /** Access and manipulation of a currently-active transaction. */ public interface Transaction { /** * @return The connection for the transaction. */ Connection connection(); /** * @param The context type. Coerced on read. * @return A {@link TransactionManager}-specific object representing the context of this * transaction. Intended for use with {@link TransactionManager} implementations that support * explicitly-passed transaction context injection into method arguments. */ default T context() { return null; } /** * Creates a prepared statement which will be cached and re-used within a transaction. Any batch * on these statements is executed before the transaction is committed, and automatically closed. * * @param sql The SQL statement * @return The statement. */ PreparedStatement prepareBatchStatement(String sql); /** * Will be called to perform work immediately after the current transaction is committed. This * should occur in the same thread and will generally not be long-lasting. * * @param runnable The code to run post-commit. */ void addPostCommitHook(Runnable runnable); } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/TransactionContextPlaceholder.java ================================================ package com.gruelbox.transactionoutbox; /** * Marker for {@link Invocation} arguments holding transaction context. These will be rehydrated * with the real context type at runtime. */ interface TransactionContextPlaceholder {} ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/TransactionManager.java ================================================ package com.gruelbox.transactionoutbox; import static com.gruelbox.transactionoutbox.spi.Utils.uncheck; import static com.gruelbox.transactionoutbox.spi.Utils.uncheckedly; import java.lang.reflect.Method; import javax.sql.DataSource; /** * Key interface giving {@link TransactionOutbox} access to JDBC. * *

In practice, most implementations should extend {@link ThreadLocalContextTransactionManager} * or {@link ParameterContextTransactionManager}. */ public interface TransactionManager { /** * Creates a simple transaction manager which uses the specified {@link DataSource} to source * connections. A new connection is requested for each transaction. * *

Transactions will be solely controlled through {@link TransactionManager}, so this may be * suitable for new applications with no other transaction management. Otherwise, a custom {@link * TransactionManager} implementation should be used. * * @param dataSource The data source. * @return The transaction manager. */ static ThreadLocalContextTransactionManager fromDataSource(DataSource dataSource) { return SimpleTransactionManager.builder() .connectionProvider(DataSourceConnectionProvider.builder().dataSource(dataSource).build()) .build(); } /** * Creates a simple transaction manager which uses the specified connection details to request a * new connection from the {@link java.sql.DriverManager} every time a new transaction starts. * *

Transactions will be solely controlled through {@link TransactionManager}, and without * pooling, performance will be poor. Generally, {@link #fromDataSource(DataSource)} using a * pooling {@code DataSource} such as that provided by Hikari is preferred. * * @param driverClass The driver class name (e.g. {@code com.mysql.cj.jdbc.Driver}). * @param url The JDBC url. * @param username The username. * @param password The password. * @return The transaction manager. */ static ThreadLocalContextTransactionManager fromConnectionDetails( String driverClass, String url, String username, String password) { return SimpleTransactionManager.builder() .connectionProvider( DriverConnectionProvider.builder() .driverClassName(driverClass) .url(url) .user(username) .password(password) .build()) .build(); } /** * Should do any work necessary to start a (new) transaction, call {@code runnable} and then * either commit on success or rollback on failure, flushing and closing any prepared statements * prior to a commit and firing post commit hooks immediately afterwards * * @param runnable Code which must be called while the transaction is active.. */ default void inTransaction(Runnable runnable) { uncheck(() -> inTransactionReturnsThrows(ThrowingTransactionalSupplier.fromRunnable(runnable))); } /** * Should do any work necessary to start a (new) transaction, call {@code runnable} and then * either commit on success or rollback on failure, flushing and closing any prepared statements * prior to a commit and firing post commit hooks immediately afterwards * * @param work Code which must be called while the transaction is active.. */ default void inTransaction(TransactionalWork work) { uncheck(() -> inTransactionReturnsThrows(ThrowingTransactionalSupplier.fromWork(work))); } /** * Should do any work necessary to start a (new) transaction, call {@code runnable} and then * either commit on success or rollback on failure, flushing and closing any prepared statements * prior to a commit and firing post commit hooks immediately afterwards. * * @param The type returned. * @param supplier Code which must be called while the transaction is active. * @return The result of {@code supplier}. */ default T inTransactionReturns(TransactionalSupplier supplier) { return uncheckedly( () -> inTransactionReturnsThrows(ThrowingTransactionalSupplier.fromSupplier(supplier))); } /** * Should do any work necessary to start a (new) transaction, call {@code runnable} and then * either commit on success or rollback on failure, flushing and closing any prepared statements * prior to a commit and firing post commit hooks immediately afterwards. * * @param work Code which must be called while the transaction is active. * @param The exception type. * @throws E If any exception is thrown by {@link Runnable}. */ @SuppressWarnings("SameReturnValue") default void inTransactionThrows(ThrowingTransactionalWork work) throws E { inTransactionReturnsThrows(ThrowingTransactionalSupplier.fromWork(work)); } /** * Should do any work necessary to start a (new) transaction, call {@code work} and then either * commit on success or rollback on failure, flushing and closing any prepared statements prior to * a commit and firing post commit hooks immediately afterwards. * * @param The type returned. * @param work Code which must be called while the transaction is active. * @param The exception type. * @return The result of {@code supplier}. * @throws E If any exception is thrown by {@link Runnable}. */ T inTransactionReturnsThrows(ThrowingTransactionalSupplier work) throws E; /** * All transaction managers need to be able to take a method call at the time it is scheduled and * determine the {@link Transaction} to use to pass to {@link Persistor} and save the request. * They can do this either by examining some current application state or by parsing the method * and arguments. * * @param method The method called. * @param args The method arguments. * @return The extracted transaction and any modifications to the method and arguments. */ TransactionalInvocation extractTransaction(Method method, Object[] args); /** * Makes any modifications to an invocation at runtime necessary to inject the current transaction * or transaction context. * * @param invocation The invocation. * @param transaction The transaction that the invocation will be run in. * @return The modified invocation. */ Invocation injectTransaction(Invocation invocation, Transaction transaction); } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/TransactionOutbox.java ================================================ package com.gruelbox.transactionoutbox; import java.time.Clock; import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.concurrent.Executor; import java.util.function.Supplier; import lombok.ToString; import org.slf4j.MDC; import org.slf4j.event.Level; /** * An implementation of the Transactional Outbox * pattern for Java. See README for * usage instructions. */ public interface TransactionOutbox { /** * @return A builder for creating a new instance of {@link TransactionOutbox}. */ static TransactionOutboxBuilder builder() { return TransactionOutboxImpl.builder(); } /** * Performs initial setup, making the instance usable. If {@link * TransactionOutboxBuilder#initializeImmediately(boolean)} is true, which is the default, this * method is called automatically when the instance is constructed. */ void initialize(); /** * The main entry point for submitting new transaction outbox tasks. * *

Returns a proxy of {@code T} which, when called, will instantly return and schedule a call * of the real method to occur after the current transaction is committed (as such a * transaction needs to be active and accessible from the transaction manager supplied to {@link * TransactionOutboxBuilder#transactionManager(TransactionManager)}), * *

Usage: * * 

transactionOutbox.schedule(MyService.class)
   *   .runMyMethod("with", "some", "arguments");
* *

This will write a record to the database using the supplied {@link Persistor} and {@link * Instantiator}, using the current database transaction, which will get rolled back if the rest * of the transaction is, and thus never processed. However, if the transaction is committed, the * real method will be called immediately afterwards using the submitter supplied to {@link * TransactionOutboxBuilder#submitter(Submitter)}. Should that fail, the call will be reattempted * whenever {@link #flush()} is called, provided at least supplied {@link * TransactionOutboxBuilder#attemptFrequency(Duration)} has passed since the time the task was * last attempted. * * @param clazz The class to proxy. * @param The type to proxy. * @return The proxy of {@code T}. */ T schedule(Class clazz); /** * Starts building a schedule request with parameterization. See {@link * ParameterizedScheduleBuilder#schedule(Class)} for more information. * * @return Builder. */ ParameterizedScheduleBuilder with(); /** * Flush in a single thread. Calls {@link #flush(Executor)} with an {@link Executor} which runs * all work in the current thread. * * @see #flush(Executor) * @return true if any work was flushed. */ default boolean flush() { return flush(Runnable::run); } /** * Identifies any stale tasks queued using {@link #schedule(Class)} (those which were queued more * than supplied {@link TransactionOutboxBuilder#attemptFrequency(Duration)} ago and have been * tried less than {@link TransactionOutboxBuilder#blockAfterAttempts(int)} )} times) and attempts * to resubmit them. * *

As long as the {@link TransactionOutboxBuilder#submitter(Submitter)} is non-blocking (e.g. * uses a bounded queue with a {@link java.util.concurrent.RejectedExecutionHandler} which throws * such as {@link java.util.concurrent.ThreadPoolExecutor.AbortPolicy}), this method will return * quickly. However, if the {@link TransactionOutboxBuilder#submitter(Submitter)} uses a bounded * queue with a blocking policy, this method could block for a long time, depending on how long * the scheduled work takes and how large {@link TransactionOutboxBuilder#flushBatchSize(int)} is. * *

Calls {@link TransactionManager#inTransactionReturns(TransactionalSupplier)} to start a new * transaction for the fetch. * *

Additionally, expires any records completed prior to the {@link * TransactionOutboxBuilder#retentionThreshold(Duration)}. * * @param executor to be used for parallelising work (note that the method overall is blocking and * this is solely ued for fork-join semantics). * @return true if any work was flushed. */ boolean flush(Executor executor); /** * Flushes a specific topic (or set of topics) * * @param executor to be used for parallelising work (note that the method overall is blocking and * this is solely ued for fork-join semantics). * @param topicNames the list of specific topics to flush * @return true if any work was flushed */ default boolean flushTopics(Executor executor, String... topicNames) { return flushTopics(executor, Arrays.asList(topicNames)); } /** * Flushes a specific topic (or set of topics) * * @param executor to be used for parallelising work (note that the method overall is blocking and * this is solely ued for fork-join semantics). * @param topicNames the list of specific topics to flush * @return true if any work was flushed */ boolean flushTopics(Executor executor, List topicNames); /** * Unblocks a blocked entry and resets the attempt count so that it will be retried again. * Requires an active transaction and a transaction manager that supports thread local context. * * @param entryId The entry id. * @return True if the request to unblock the entry was successful. May return false if another * thread unblocked the entry first. */ boolean unblock(String entryId); /** * Clears a failed entry of its failed state and resets the attempt count so that it will be * retried again. Requires an active transaction and a transaction manager that supports supplied * context. * * @param entryId The entry id. * @param transactionContext The transaction context ({@link TransactionManager} implementation * specific). * @return True if the request to unblock the entry was successful. May return false if another * thread unblocked the entry first. */ @SuppressWarnings("unused") boolean unblock(String entryId, Object transactionContext); /** * Processes an entry immediately in the current thread. Intended for use in custom * implementations of {@link Submitter} and should not generally otherwise be called. * * @param entry The entry. */ @SuppressWarnings("WeakerAccess") void processNow(TransactionOutboxEntry entry); /** Builder for {@link TransactionOutbox}. */ @ToString abstract class TransactionOutboxBuilder { protected TransactionManager transactionManager; protected Instantiator instantiator; protected Submitter submitter; protected Duration attemptFrequency; protected int blockAfterAttempts; protected int flushBatchSize; protected Supplier clockProvider; protected TransactionOutboxListener listener; protected Persistor persistor; protected Level logLevelTemporaryFailure; protected Boolean serializeMdc; protected Duration retentionThreshold; protected Boolean initializeImmediately; protected TransactionOutboxBuilder() {} /** * @param transactionManager Provides {@link TransactionOutbox} with the ability to start, * commit and roll back transactions as well as interact with running transactions started * outside. * @return Builder. */ public TransactionOutboxBuilder transactionManager(TransactionManager transactionManager) { this.transactionManager = transactionManager; return this; } /** * @param instantiator Responsible for describing a class as a name and creating instances of * that class at runtime from the name. See {@link Instantiator} for more information. * Defaults to {@link Instantiator#usingReflection()}. * @return Builder. */ public TransactionOutboxBuilder instantiator(Instantiator instantiator) { this.instantiator = instantiator; return this; } /** * @param submitter Used for scheduling background work. If no submitter is specified, {@link * TransactionOutbox} will use {@link Submitter#withDefaultExecutor()}. See {@link * Submitter#withExecutor(Executor)} for more information on designing bespoke submitters * for remoting. * @return Builder. */ public TransactionOutboxBuilder submitter(Submitter submitter) { this.submitter = submitter; return this; } /** * @param attemptFrequency How often tasks should be re-attempted. This should be balanced with * {@link #flushBatchSize} and the frequency with which {@link #flush()} is called to * achieve optimum throughput. Defaults to 2 minutes. * @return Builder. */ public TransactionOutboxBuilder attemptFrequency(Duration attemptFrequency) { this.attemptFrequency = attemptFrequency; return this; } /** * @param blockAfterAttempts how many attempts a task should be retried before it is permanently * blocked. Defaults to 5. * @return Builder. */ public TransactionOutboxBuilder blockAfterAttempts(int blockAfterAttempts) { this.blockAfterAttempts = blockAfterAttempts; return this; } /** * @param flushBatchSize How many items should be attempted in each flush. This should be * balanced with {@link #attemptFrequency} and the frequency with which {@link #flush()} is * called to achieve optimum throughput. Defaults to 4096. * @return Builder. */ public TransactionOutboxBuilder flushBatchSize(int flushBatchSize) { this.flushBatchSize = flushBatchSize; return this; } /** * @param clockProvider The {@link Clock} source. Generally best left alone except when testing. * Defaults to the system clock. * @return Builder. */ public TransactionOutboxBuilder clockProvider(Supplier clockProvider) { this.clockProvider = clockProvider; return this; } /** * @param listener Event listener. Allows client code to react to tasks running, failing or * getting blocked. * @return Builder. */ public TransactionOutboxBuilder listener(TransactionOutboxListener listener) { this.listener = listener; return this; } /** * @param persistor The method {@link TransactionOutbox} uses to interact with the database. * This encapsulates all {@link TransactionOutbox} interaction with the database outside * transaction management (which is handled by the {@link TransactionManager}). Defaults to * a multi-platform SQL implementation that should not need to be changed in most cases. If * re-implementing this interface, read the documentation on {@link Persistor} carefully. * @return Builder. */ public TransactionOutboxBuilder persistor(Persistor persistor) { this.persistor = persistor; return this; } /** * @param logLevelTemporaryFailure The log level to use when logging temporary task failures. * Includes a full stack trace. Defaults to {@code WARN} level, but you may wish to reduce * it to a lower level if you consider warnings to be incidents. * @return Builder. */ public TransactionOutboxBuilder logLevelTemporaryFailure(Level logLevelTemporaryFailure) { this.logLevelTemporaryFailure = logLevelTemporaryFailure; return this; } /** * @param serializeMdc Determines whether to include any Slf4j {@link MDC} (Mapped Diagnostic * Context) in serialized invocations and recreate the state in submitted tasks. Defaults to * true. * @return Builder. */ public TransactionOutboxBuilder serializeMdc(Boolean serializeMdc) { this.serializeMdc = serializeMdc; return this; } /** * @param retentionThreshold The length of time that any request with a unique client id will be * remembered, such that if the same request is repeated within the threshold period, {@link * AlreadyScheduledException} will be thrown. * @return Builder. */ public TransactionOutboxBuilder retentionThreshold(Duration retentionThreshold) { this.retentionThreshold = retentionThreshold; return this; } /** * @param initializeImmediately If true, {@link TransactionOutbox#initialize()} is called * automatically on creation (this is the default). Set to false in environments where * structured startup means that the database should not be accessed until later. * @return Builder. */ public TransactionOutboxBuilder initializeImmediately(boolean initializeImmediately) { this.initializeImmediately = initializeImmediately; return this; } /** * Creates and initialises the {@link TransactionOutbox}. * * @return The outbox implementation. */ public abstract TransactionOutbox build(); } interface ParameterizedScheduleBuilder { /** * Specifies a unique id for the request. This defaults to {@code null}, but if non-null, will * cause the request to be retained in the database after completion for the specified {@link * TransactionOutboxBuilder#retentionThreshold(Duration)}, during which time any duplicate * requests to schedule the same request id will throw {@link AlreadyScheduledException}. This * allows tasks to be scheduled idempotently even if the request itself is not idempotent (e.g. * from a message queue listener, which can usually only work reliably on an "at least once" * basis). * * @param uniqueRequestId The unique request id. May be {@code null}, but if non-null may be a * maximum of 250 characters in length. It is advised that if these ids are client-supplied, * they be prepended with some sort of context identifier to ensure global uniqueness. * @return Builder. */ ParameterizedScheduleBuilder uniqueRequestId(String uniqueRequestId); /** * Specifies that the request should be applied in a strictly-ordered fashion within the * specified topic. * *

This is useful for a number of applications, such as feeding messages into an ordered * pipeline such as a FIFO queue or Kafka topic, or for reliable data replication, such as when * feeding a data warehouse or distributed cache. * *

Note that using this option has a number of consequences: * *

    *
  • Requests are not processed immediately when submitting a request, as normal, and are * processed by {@link TransactionOutbox#flush()} only. As a result there will be * increased delay between the source transaction being committed and the request being * processed. *
  • If a request fails, no further requests will be processed in that topic until * a subsequent retry allows the failing request to succeed, to preserve ordered * processing. This means it is possible for topics to become entirely frozen in the event * that a request fails repeatedly. For this reason, it is essential to use a {@link * TransactionOutboxListener} to watch for failing requests and investigate quickly. Note * that other topics will be unaffected. *
  • For the same reason, {@link TransactionOutboxBuilder#blockAfterAttempts} is ignored for * all requests that use this option. The only safe way to recover from a failing request * is to make the request succeed. *
  • A single topic can only be processed in single-threaded fashion, so if your requests * use a small number of topics, scalability will be affected since the degree of * parallelism will be reduced. *
  • Throughput is significantly reduced and database load increased more generally, even * with larger numbers of topics, since records are only processed one-at-a-time rather * than in batches, which is less optimised. *
  • In general, databases * are not well optimised for this sort of thing. Don't expect miracles. If you need * more throughput, you probably need to think twice about your architecture. Consider the * event sourcing * pattern, for example, where the message queue is the primary data store rather than * a secondary, and remove the need for an outbox entirely. *
* * @param topic a free-text string up to 250 characters. * @return Builder. */ ParameterizedScheduleBuilder ordered(String topic); /** * Instructs the scheduler to delay processing the task until after the specified duration. This * can be used for simple job scheduling or to introduce an asynchronous delay into chains of * tasks. * *

Note that any delay is not precise and accuracy is primarily determined by the * frequency at which {@link #flush(Executor)} or {@link #flush()} are called. Do not use this * for time-sensitive tasks, particularly if the duration exceeds {@link * TransactionOutboxBuilder#attemptFrequency(Duration)} (see more on this below). * *

A note on implementation: tasks (when {@link #ordered(String)} is not used) are normally * submitted for processing on the local JVM immediately after transaction commit. By default, * when a delay is introduced, the work is instead submitted to a {@link * java.util.concurrent.ScheduledExecutorService} for processing after the specified delay. * However, if the delay is long enough that the work would likely get picked up by a {@link * #flush()} on this JVM or another, this is pointless and wasteful. Unfortunately, we don't * know exactly how frequently {@link #flush()} will be called! To mitigate this, Any task * submitted with a delay in excess of {@link * TransactionOutboxBuilder#attemptFrequency(Duration)} will be assumed to get picked up by a * future flush. * * @param duration The minimum delay duration. * @return Builder. */ ParameterizedScheduleBuilder delayForAtLeast(Duration duration); /** * Equivalent to {@link TransactionOutbox#schedule(Class)}, but applying additional parameters * to the request as configured using {@link TransactionOutbox#with()}. * *

Usage example: * * 

transactionOutbox.with()
     * .uniqueRequestId("my-request")
     * .schedule(MyService.class)
     * .runMyMethod("with", "some", "arguments");
* * @param clazz The class to proxy. * @param The type to proxy. * @return The proxy of {@code T}. */ T schedule(Class clazz); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/TransactionOutboxEntry.java ================================================ package com.gruelbox.transactionoutbox; import static java.util.stream.Collectors.joining; import com.gruelbox.transactionoutbox.spi.Utils; import java.time.Instant; import java.util.Arrays; import lombok.*; import lombok.experimental.SuperBuilder; /** * Internal representation of a {@link TransactionOutbox} task. Generally only directly of interest * to implementers of SPIs such as {@link Persistor} or {@link Submitter}. */ @SuperBuilder(toBuilder = true) @EqualsAndHashCode @ToString public class TransactionOutboxEntry implements Validatable { /** * @param id The id of the record. Usually a UUID. * @return The id of the record. Usually a UUID. */ @SuppressWarnings("JavaDoc") @Getter private final String id; /** * @param uniqueRequestId A unique, client-supplied key for the entry. If supplied, it must be * globally unique */ @SuppressWarnings("JavaDoc") @Getter private final String uniqueRequestId; /** * @param topic An optional scope for ordered sequencing. */ @SuppressWarnings("JavaDoc") @Getter private final String topic; /** * @param sequence The ordered sequence within the {@code topic}. */ @SuppressWarnings("JavaDoc") @Getter @Setter private Long sequence; /** * @param invocation The method invocation to perform. * @return The method invocation to perform. */ @SuppressWarnings("JavaDoc") @Getter @Setter(AccessLevel.PACKAGE) private Invocation invocation; /** * @param lastAttemptTime The timestamp at which the task was last processed. * @return The timestamp at which the task was last processed. */ @SuppressWarnings("JavaDoc") @Getter @Setter private Instant lastAttemptTime; /** * @param nextAttemptTime The timestamp after which the task is available for re-attempting. * @return The timestamp after which the task is available for re-attempting. */ @SuppressWarnings("JavaDoc") @Getter @Setter private Instant nextAttemptTime; /** * @param attempts The number of unsuccessful attempts so far made to run the task. * @return The number of unsuccessful attempts so far made to run the task. */ @SuppressWarnings("JavaDoc") @Getter @Setter private int attempts; /** * @param blocked True if the task has exceeded the configured maximum number of attempts. * @return True if the task has exceeded the configured maximum number of attempts. */ @SuppressWarnings("JavaDoc") @Getter @Setter private boolean blocked; /** * @param processed True if the task has been processed but has been retained to prevent duplicate * requests. * @return True if the task has been processed but has been retained to prevent * duplicate * requests. */ @SuppressWarnings("JavaDoc") @Getter @Setter private boolean processed; /** * @param version The optimistic locking version. Monotonically increasing with each update. * @return The optimistic locking version. Monotonically increasing with each update. */ @SuppressWarnings("JavaDoc") @Getter @Setter private int version; @EqualsAndHashCode.Exclude @ToString.Exclude private volatile boolean initialized; @EqualsAndHashCode.Exclude @ToString.Exclude private String description; /** * @return A textual description of the task. */ public String description() { if (!this.initialized) { synchronized (this) { if (!this.initialized) { String description = String.format( "%s.%s(%s) [%s]%s%s", invocation.getClassName(), invocation.getMethodName(), invocation.getArgs() == null ? null : Arrays.stream(invocation.getArgs()) .map(Utils::stringify) .collect(joining(", ")), id, uniqueRequestId == null ? "" : " uid=[" + uniqueRequestId + "]", topic == null ? "" : " seq=[" + topic + "/" + sequence + "]"); this.description = description; this.initialized = true; return description; } } } return this.description; } @Override public void validate(Validator validator) { validator.notNull("id", id); validator.nullOrNotBlank("uniqueRequestId", uniqueRequestId); validator.nullOrNotBlank("topic", topic); validator.notNull("invocation", invocation); validator.positiveOrZero("attempts", attempts); validator.positiveOrZero("version", version); validator.isTrue("topic", !"*".equals(topic), "Topic may not be *"); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/TransactionOutboxImpl.java ================================================ package com.gruelbox.transactionoutbox; import static com.gruelbox.transactionoutbox.spi.Utils.logAtLevel; import static com.gruelbox.transactionoutbox.spi.Utils.uncheckedly; import static java.time.temporal.ChronoUnit.MILLIS; import static java.time.temporal.ChronoUnit.MINUTES; import com.gruelbox.transactionoutbox.spi.ProxyFactory; import com.gruelbox.transactionoutbox.spi.Utils; import java.lang.reflect.InvocationTargetException; import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.UUID; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import java.util.function.Supplier; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.ToString; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import org.slf4j.MDC; import org.slf4j.event.Level; @Slf4j @RequiredArgsConstructor(access = AccessLevel.PRIVATE) final class TransactionOutboxImpl implements TransactionOutbox, Validatable { /** * Used in tests to ensure that the full serialization/deserialization loop is done every time * work is processed. This prevents tests passing which would fail if the work were delayed and * handled after being loaded from disk. */ static final AtomicBoolean FORCE_SERIALIZE_AND_DESERIALIZE_BEFORE_USE = new AtomicBoolean(); private final TransactionManager transactionManager; private final Persistor persistor; private final Instantiator instantiator; private final Submitter submitter; private final Duration attemptFrequency; private final Level logLevelTemporaryFailure; private final int blockAfterAttempts; private final int flushBatchSize; private final Supplier clockProvider; private final TransactionOutboxListener listener; private final boolean serializeMdc; private final Validator validator; private final Duration retentionThreshold; private final AtomicBoolean initialized = new AtomicBoolean(); private final ProxyFactory proxyFactory = new ProxyFactory(); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); @Override public void validate(Validator validator) { validator.notNull("transactionManager", transactionManager); validator.valid("persistor", persistor); validator.valid("instantiator", instantiator); validator.valid("submitter", submitter); validator.notNull("attemptFrequency", attemptFrequency); validator.notNull("logLevelTemporaryFailure", logLevelTemporaryFailure); validator.min("blockAfterAttempts", blockAfterAttempts, 1); validator.min("flushBatchSize", flushBatchSize, 1); validator.notNull("clockProvider", clockProvider); validator.notNull("listener", listener); validator.notNull("retentionThreshold", retentionThreshold); } static TransactionOutboxBuilder builder() { return new TransactionOutboxBuilderImpl(); } @Override public void initialize() { if (initialized.compareAndSet(false, true)) { try { persistor.migrate(transactionManager); } catch (Exception e) { initialized.set(false); throw e; } } } @Override public T schedule(Class clazz) { return schedule(clazz, null, null, null); } @Override public ParameterizedScheduleBuilder with() { return new ParameterizedScheduleBuilderImpl(); } private boolean doFlush(Function> batchSource) { var batch = transactionManager.inTransactionReturns( transaction -> { var entries = batchSource.apply(transaction); List result = new ArrayList<>(entries.size()); for (var entry : entries) { log.debug("Triggering {}", entry.description()); try { pushBack(transaction, entry); result.add(entry); } catch (OptimisticLockException e) { log.debug("Beaten to optimistic lock on {}", entry.description()); } } return result; }); log.debug("Got batch of {}", batch.size()); batch.forEach(this::submitNow); log.debug("Submitted batch"); return !batch.isEmpty(); } @Override public boolean flush(Executor executor) { if (!initialized.get()) { throw new IllegalStateException("Not initialized"); } Instant now = clockProvider.get().instant(); List> futures = new ArrayList<>(); futures.add( CompletableFuture.supplyAsync( () -> { log.debug("Flushing stale tasks"); return doFlush( tx -> uncheckedly(() -> persistor.selectBatch(tx, flushBatchSize, now))); }, executor)); futures.add( CompletableFuture.runAsync(() -> expireIdempotencyProtection(now), executor) .thenApply(it -> false)); futures.add( CompletableFuture.supplyAsync( () -> { log.debug("Flushing topics"); return doFlush( tx -> uncheckedly(() -> persistor.selectNextInTopics(tx, flushBatchSize, now))); }, executor)); return futures.stream() .reduce((f1, f2) -> f1.thenCombine(f2, (d1, d2) -> d1 || d2)) .map(CompletableFuture::join) .orElse(false); } @Override public boolean flushTopics(Executor executor, List topicNames) { if (!initialized.get()) { throw new IllegalStateException("Not initialized"); } Instant now = clockProvider.get().instant(); log.debug("Flushing selected topics {}", topicNames); return doFlush( tx -> uncheckedly( () -> persistor.selectNextInSelectedTopics(tx, topicNames, flushBatchSize, now))); } private void expireIdempotencyProtection(Instant now) { long totalRecordsDeleted = 0; int recordsDeleted; do { recordsDeleted = transactionManager.inTransactionReturns( tx -> uncheckedly(() -> persistor.deleteProcessedAndExpired(tx, flushBatchSize, now))); totalRecordsDeleted += recordsDeleted; } while (recordsDeleted > 0); if (totalRecordsDeleted > 0) { String duration = String.format( "%dd:%02dh:%02dm:%02ds", retentionThreshold.toDaysPart(), retentionThreshold.toHoursPart(), retentionThreshold.toMinutesPart(), retentionThreshold.toSecondsPart()); log.info( "Expired idempotency protection on {} requests completed more than {} ago", totalRecordsDeleted, duration); } else { log.debug("No records found to delete as of {}", now); } } @Override public boolean unblock(String entryId) { if (!initialized.get()) { throw new IllegalStateException("Not initialized"); } if (!(transactionManager instanceof ThreadLocalContextTransactionManager)) { throw new UnsupportedOperationException( "This method requires a ThreadLocalContextTransactionManager"); } log.info("Unblocking entry {} for retry.", entryId); try { return ((ThreadLocalContextTransactionManager) transactionManager) .requireTransactionReturns(tx -> persistor.unblock(tx, entryId)); } catch (Exception e) { throw (RuntimeException) Utils.uncheckAndThrow(e); } } @Override @SuppressWarnings({"unchecked", "rawtypes"}) public boolean unblock(String entryId, Object transactionContext) { if (!initialized.get()) { throw new IllegalStateException("Not initialized"); } if (!(transactionManager instanceof ParameterContextTransactionManager)) { throw new UnsupportedOperationException( "This method requires a ParameterContextTransactionManager"); } log.info("Unblocking entry {} for retry", entryId); try { if (transactionContext instanceof Transaction) { return persistor.unblock((Transaction) transactionContext, entryId); } Transaction transaction = ((ParameterContextTransactionManager) transactionManager) .transactionFromContext(transactionContext); return persistor.unblock(transaction, entryId); } catch (Exception e) { throw (RuntimeException) Utils.uncheckAndThrow(e); } } private T schedule( Class clazz, String uniqueRequestId, String topic, Duration delayForAtLeast) { if (!initialized.get()) { throw new IllegalStateException("Not initialized"); } return proxyFactory.createProxy( clazz, (method, args) -> uncheckedly( () -> { var extracted = transactionManager.extractTransaction(method, args); TransactionOutboxEntry entry = newEntry( extracted.getClazz(), extracted.getMethodName(), extracted.getParameters(), extracted.getArgs(), uniqueRequestId, topic); if (delayForAtLeast != null) { entry.setNextAttemptTime(entry.getNextAttemptTime().plus(delayForAtLeast)); } validator.validate(entry); persistor.save(extracted.getTransaction(), entry); extracted .getTransaction() .addPostCommitHook( () -> { listener.scheduled(entry); if (entry.getTopic() != null) { log.debug("Queued {} in topic {}", entry.description(), topic); } else if (delayForAtLeast == null) { submitNow(entry); log.debug( "Scheduled {} for post-commit execution", entry.description()); } else if (delayForAtLeast.compareTo(attemptFrequency) < 0) { scheduler.schedule( () -> submitNow(entry), delayForAtLeast.toMillis(), TimeUnit.MILLISECONDS); log.info( "Scheduled {} for post-commit execution after at least {}", entry.description(), delayForAtLeast); } else { log.info( "Queued {} for execution after at least {}", entry.description(), delayForAtLeast); } }); return null; })); } private void submitNow(TransactionOutboxEntry entry) { submitter.submit(entry, this::processNow); } @Override @SuppressWarnings("WeakerAccess") public void processNow(TransactionOutboxEntry entry) { listener.wrapInvocationAndInit( new TransactionOutboxListener.Invocator() { @Override public void run() { processNowInternal(entry); } @Override public Invocation getInvocation() { return entry.getInvocation(); } }); } private void processNowInternal(TransactionOutboxEntry entry) { initialize(); Boolean success = null; try { success = entry .getInvocation() .withinMDC( () -> transactionManager.inTransactionReturnsThrows( tx -> { if (!persistor.lock(tx, entry)) { return false; } log.info("Processing {}", entry.description()); invoke(entry, tx); if (entry.getUniqueRequestId() == null) { persistor.delete(tx, entry); } else { log.debug( "Deferring deletion of {} by {}", entry.description(), retentionThreshold); entry.setProcessed(true); entry.setLastAttemptTime(Instant.now(clockProvider.get())); entry.setNextAttemptTime(after(retentionThreshold)); persistor.update(tx, entry); } return true; })); } catch (InvocationTargetException e) { updateAttemptCount(entry, e.getCause()); } catch (Exception e) { updateAttemptCount(entry, e); } if (success != null) { if (success) { log.info("Processed {}", entry.description()); listener.success(entry); } else { log.debug("Skipped task {} - may be locked or already processed", entry.getId()); } } } private void invoke(TransactionOutboxEntry entry, Transaction transaction) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { Object instance = instantiator.getInstance(entry.getInvocation().getClassName()); log.debug("Created instance {}", instance); transactionManager .injectTransaction(entry.getInvocation(), transaction) .invoke(instance, listener); } private TransactionOutboxEntry newEntry( Class clazz, String methodName, Class[] params, Object[] args, String uniqueRequestId, String topic) { var invocation = new Invocation( instantiator.getName(clazz), methodName, params, args, serializeMdc && (MDC.getMDCAdapter() != null) ? MDC.getCopyOfContextMap() : null, listener.extractSession()); if (FORCE_SERIALIZE_AND_DESERIALIZE_BEFORE_USE.get()) { invocation = persistor.serializeAndDeserialize(invocation); } return TransactionOutboxEntry.builder() .id(UUID.randomUUID().toString()) .invocation(invocation) .lastAttemptTime(null) .nextAttemptTime(clockProvider.get().instant()) .uniqueRequestId(uniqueRequestId) .topic(topic) .build(); } private void pushBack(Transaction transaction, TransactionOutboxEntry entry) throws OptimisticLockException { try { entry.setLastAttemptTime(clockProvider.get().instant()); entry.setNextAttemptTime(after(attemptFrequency)); validator.validate(entry); persistor.update(transaction, entry); } catch (OptimisticLockException e) { throw e; } catch (Exception e) { Utils.uncheckAndThrow(e); } } private Instant after(Duration duration) { return clockProvider.get().instant().plus(duration).truncatedTo(MILLIS); } private void updateAttemptCount(TransactionOutboxEntry entry, Throwable cause) { try { entry.setAttempts(entry.getAttempts() + 1); var blocked = (entry.getTopic() == null) && (entry.getAttempts() >= blockAfterAttempts); entry.setBlocked(blocked); transactionManager.inTransactionThrows(tx -> pushBack(tx, entry)); listener.failure(entry, cause); if (blocked) { log.error( "Blocking failing entry {} after {} attempts: {}", entry.getId(), entry.getAttempts(), entry.description(), cause); listener.blocked(entry, cause); } else { logAtLevel( log, logLevelTemporaryFailure, "Temporarily failed to process entry {} : {}", entry.getId(), entry.description(), cause); } } catch (Exception e) { log.error( "Failed to update attempt count for {}. It may be retried more times than expected.", entry.description(), e); } } @ToString static class TransactionOutboxBuilderImpl extends TransactionOutboxBuilder { TransactionOutboxBuilderImpl() { super(); } public TransactionOutboxImpl build() { Validator validator = new Validator(this.clockProvider); TransactionOutboxImpl impl = new TransactionOutboxImpl( transactionManager, persistor, Utils.firstNonNull(instantiator, Instantiator::usingReflection), Utils.firstNonNull(submitter, Submitter::withDefaultExecutor), Utils.firstNonNull(attemptFrequency, () -> Duration.of(2, MINUTES)), Utils.firstNonNull(logLevelTemporaryFailure, () -> Level.WARN), blockAfterAttempts < 1 ? 5 : blockAfterAttempts, flushBatchSize < 1 ? 4096 : flushBatchSize, clockProvider == null ? Clock::systemDefaultZone : clockProvider, Utils.firstNonNull(listener, () -> TransactionOutboxListener.EMPTY), serializeMdc == null || serializeMdc, validator, retentionThreshold == null ? Duration.ofDays(7) : retentionThreshold); validator.validate(impl); if (initializeImmediately == null || initializeImmediately) { impl.initialize(); } return impl; } } @Accessors(fluent = true, chain = true) @Setter private class ParameterizedScheduleBuilderImpl implements ParameterizedScheduleBuilder { private String uniqueRequestId; private String ordered; private Duration delayForAtLeast; @Override public T schedule(Class clazz) { if (uniqueRequestId != null && uniqueRequestId.length() > 250) { throw new IllegalArgumentException("uniqueRequestId may be up to 250 characters"); } return TransactionOutboxImpl.this.schedule(clazz, uniqueRequestId, ordered, delayForAtLeast); } } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/TransactionOutboxListener.java ================================================ package com.gruelbox.transactionoutbox; import java.lang.reflect.InvocationTargetException; import java.util.HashMap; import java.util.Map; /** A listener for events fired by {@link TransactionOutbox}. */ public interface TransactionOutboxListener { TransactionOutboxListener EMPTY = new TransactionOutboxListener() {}; /** * Fired when a transaction outbox task is scheduled. * *

This event is not guaranteed to fire in the event of a JVM failure or power loss. It is * fired after the commit to the database adding the scheduled task but before the task * is submitted for processing. It will, except in extreme circumstances (although this is not * guaranteed), fire prior to any subsequent {@link #success(TransactionOutboxEntry)} or {@link * #failure(TransactionOutboxEntry, Throwable)}. * * @param entry The outbox entry scheduled. */ default void scheduled(TransactionOutboxEntry entry) { // No-op } /** * Implement this method to intercept and decorate all outbox invocations. In general, you should * call {@code invocation.run()} which actually calls the underlying method, unless you are * deliberately trying to suppress the method call. * *

This method is called immediately before your method is invoked. A fair bit of work has * already been done by this point (MDC initialised, transaction started, database record locked * etc) so it's a good place to do database-related things, but a poor place to initialise things * like session state (such as OTEL traces). For that, use {@link * #wrapInvocationAndInit(Invocator)}. * * @param invocator A runnable which performs the work of the scheduled task. * @throws IllegalAccessException If thrown by the method invocation. * @throws IllegalArgumentException If thrown by the method invocation. * @throws InvocationTargetException If thrown by the method invocation. */ default void wrapInvocation(Invocator invocator) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { invocator.run(); } /** * Wraps an entire invocation, including the work do obtain a database lock. This is a good place * to initialise session state from {@link Invocation#getSession()} (using {@link * Invocator#getInvocation()}) since then all subsequent database activity is performed within * that session. * *

NOTE that there is no guarantee that your method will actually be invoked at this point. * There is no active database transaction, the database record hasn't been locked and important * checks haven't been performed. This is intended purely for state management. * * @param invocator A runnable which performs the work of the scheduled task. */ default void wrapInvocationAndInit(Invocator invocator) { try { invocator.run(); } catch (IllegalAccessException | InvocationTargetException e) { throw new RuntimeException(e); } } interface Invocator { void run() throws IllegalAccessException, IllegalArgumentException, InvocationTargetException; default void runUnchecked() { try { run(); } catch (IllegalAccessException | InvocationTargetException e) { throw new UncheckedException(e); } } /** * @return The full {@link Invocation} object, for use in {@link #wrapInvocation(Invocator)} and * {@link #wrapInvocationAndInit(Invocator)}. */ Invocation getInvocation(); } /** * Fired when a transaction outbox task is successfully completed and recorded as such in * the database such that it will not be re-attempted. Note that: * *

    *
  • {@link TransactionOutbox} uses "at least once" semantics, so the actual processing of a * task may complete any number of times before this event is fired. *
  • This event is not guaranteed to fire in the event of a JVM failure or power loss. It is * fired after the commit to the database removing the completed task and all bets * are off after this point. *
* * @param entry The outbox entry completed. */ default void success(TransactionOutboxEntry entry) { // No-op } /** * Fired when a transaction outbox task fails. This may occur multiple times until the maximum * number of retries, at which point this will be fired and then {@link * #blocked(TransactionOutboxEntry, Throwable)}. This event is not guaranteed to fire in the event * of a JVM failure or power loss. It is fired after the commit to the database marking * the task as failed. * * @param entry The outbox entry failed. * @param cause The cause of the most recent failure. */ default void failure(TransactionOutboxEntry entry, Throwable cause) { // No-op } /** * Fired when a transaction outbox task has passed the maximum number of retries and has been * blocked. This event is not guaranteed to fire in the event of a JVM failure or power loss. It * is fired after the commit to the database marking the task as blocked. * * @param entry The outbox entry to be marked as blocked. * @param cause The cause of the most recent failure. */ default void blocked(TransactionOutboxEntry entry, Throwable cause) { // No-op } /** * Implement this to provide session state that you want to include with a persisted request. This * is a free-form {@link Map} which you can then read in {@link #wrapInvocation(Invocator)} using * {@link Invocator#getInvocation()}. * * @return Your session information. */ default Map extractSession() { return null; } /** * Chains this listener with another and returns the result. * * @param other The other listener. It will always be called after this one. * @return The combined listener. */ default TransactionOutboxListener andThen(TransactionOutboxListener other) { var self = this; return new TransactionOutboxListener() { @Override public void scheduled(TransactionOutboxEntry entry) { self.scheduled(entry); other.scheduled(entry); } @Override public void wrapInvocation(Invocator invocator) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { self.wrapInvocation( new Invocator() { @Override public void run() throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { other.wrapInvocation(invocator); } @Override public Invocation getInvocation() { return invocator.getInvocation(); } }); } @Override public void wrapInvocationAndInit(Invocator invocator) { self.wrapInvocationAndInit( new Invocator() { @Override public void run() { other.wrapInvocationAndInit(invocator); } @Override public Invocation getInvocation() { return invocator.getInvocation(); } }); } @Override public Map extractSession() { var mine = self.extractSession(); var theirs = other.extractSession(); if (mine == null) return theirs; if (theirs == null) return mine; Map result = new HashMap<>(mine); result.putAll(theirs); return result; } @Override public void success(TransactionOutboxEntry entry) { self.success(entry); other.success(entry); } @Override public void failure(TransactionOutboxEntry entry, Throwable cause) { self.failure(entry, cause); other.failure(entry, cause); } @Override public void blocked(TransactionOutboxEntry entry, Throwable cause) { self.blocked(entry, cause); other.blocked(entry, cause); } }; } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/TransactionalInvocation.java ================================================ package com.gruelbox.transactionoutbox; import lombok.Value; /** * Describes a method invocation along with the transaction scope in which it should be performed. */ @Value public class TransactionalInvocation { Class clazz; String methodName; Class[] parameters; Object[] args; Transaction transaction; } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/TransactionalSupplier.java ================================================ package com.gruelbox.transactionoutbox; @FunctionalInterface public interface TransactionalSupplier { static TransactionalSupplier fromWork(TransactionalWork work) { return transaction -> { work.doWork(transaction); return null; }; } T doWork(Transaction transaction); } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/TransactionalWork.java ================================================ package com.gruelbox.transactionoutbox; @FunctionalInterface public interface TransactionalWork { void doWork(Transaction transaction); } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/UncheckedException.java ================================================ package com.gruelbox.transactionoutbox; /** A wrapped {@link Exception} where unchecked exceptions are caught and propagated as runtime. */ @SuppressWarnings("WeakerAccess") public class UncheckedException extends RuntimeException { public UncheckedException(Throwable cause) { super(cause); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/Validatable.java ================================================ package com.gruelbox.transactionoutbox; interface Validatable { void validate(Validator validator); } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/Validator.java ================================================ package com.gruelbox.transactionoutbox; import java.time.Clock; import java.util.function.Supplier; class Validator { private final String path; private final Supplier clockProvider; Validator(Supplier clockProvider) { this.clockProvider = clockProvider; this.path = ""; } private Validator(String className, Validator validator) { this.clockProvider = validator.clockProvider; this.path = className; } public void validate(Validatable validatable) { validatable.validate(new Validator(validatable.getClass().getSimpleName(), this)); } public void valid(String propertyName, Object object) { notNull(propertyName, object); if (!(object instanceof Validatable)) { return; } ((Validatable) object) .validate(new Validator(path.isEmpty() ? propertyName : (path + "." + propertyName), this)); } public void notNull(String propertyName, Object object) { if (object == null) { error(propertyName, "may not be null"); } } public void isTrue(String propertyName, boolean condition, String message, Object... args) { if (!condition) { error(propertyName, String.format(message, args)); } } public void nullOrNotBlank(String propertyName, String object) { if (object != null && object.isEmpty()) { error(propertyName, "may be either null or non-blank"); } } public void notBlank(String propertyName, String object) { notNull(propertyName, object); if (object.isEmpty()) { error(propertyName, "may not be blank"); } } public void positiveOrZero(String propertyName, int object) { min(propertyName, object, 0); } public void min(String propertyName, int object, int minimumValue) { if (object < minimumValue) { error(propertyName, "must be greater than " + minimumValue); } } private void error(String propertyName, String message) { throw new IllegalArgumentException( (path.isEmpty() ? "" : path + ".") + propertyName + " " + message); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/spi/AbstractFullyQualifiedNameInstantiator.java ================================================ package com.gruelbox.transactionoutbox.spi; import static com.gruelbox.transactionoutbox.spi.Utils.uncheckedly; import com.gruelbox.transactionoutbox.Instantiator; import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; /** * Abstract {@link Instantiator} implementation which simplifies the creation of implementations * which instantiate based on the clazz FQN. */ @Slf4j @SuperBuilder @NoArgsConstructor(access = AccessLevel.PROTECTED) public abstract class AbstractFullyQualifiedNameInstantiator implements Instantiator { @Override public final String getName(Class clazz) { return clazz.getName(); } @Override public final Object getInstance(String name) { log.debug("Getting class by name [{}]", name); return createInstance(uncheckedly(() -> Class.forName(name))); } protected abstract Object createInstance(Class clazz); } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/spi/AbstractThreadLocalTransactionManager.java ================================================ package com.gruelbox.transactionoutbox.spi; import com.gruelbox.transactionoutbox.*; import java.util.Deque; import java.util.LinkedList; import java.util.Optional; import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.experimental.SuperBuilder; import lombok.extern.slf4j.Slf4j; @Slf4j @SuperBuilder @NoArgsConstructor(access = AccessLevel.PROTECTED) public abstract class AbstractThreadLocalTransactionManager implements ThreadLocalContextTransactionManager { private final ThreadLocal> transactions = ThreadLocal.withInitial(LinkedList::new); @Override public final void inTransaction(Runnable runnable) { ThreadLocalContextTransactionManager.super.inTransaction(runnable); } @Override public final void inTransaction(TransactionalWork work) { ThreadLocalContextTransactionManager.super.inTransaction(work); } @Override public final T inTransactionReturns(TransactionalSupplier supplier) { return ThreadLocalContextTransactionManager.super.inTransactionReturns(supplier); } @Override public final void inTransactionThrows(ThrowingTransactionalWork work) throws E { ThreadLocalContextTransactionManager.super.inTransactionThrows(work); } @Override public T requireTransactionReturns( ThrowingTransactionalSupplier work) throws E, NoTransactionActiveException { return work.doWork(peekTransaction().orElseThrow(NoTransactionActiveException::new)); } public final TX pushTransaction(TX transaction) { transactions.get().push(transaction); return transaction; } public final TX popTransaction() { TX result = transactions.get().pop(); if (transactions.get().isEmpty()) { transactions.remove(); } return result; } public Optional peekTransaction() { return Optional.ofNullable(transactions.get().peek()); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/spi/ProxyFactory.java ================================================ package com.gruelbox.transactionoutbox.spi; import com.gruelbox.transactionoutbox.MissingOptionalDependencyException; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.util.function.BiFunction; import lombok.extern.slf4j.Slf4j; import net.bytebuddy.ByteBuddy; import net.bytebuddy.TypeCache; import net.bytebuddy.TypeCache.Sort; import net.bytebuddy.description.modifier.Visibility; import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; import net.bytebuddy.implementation.InvocationHandlerAdapter; import net.bytebuddy.matcher.ElementMatchers; import org.objenesis.Objenesis; import org.objenesis.ObjenesisStd; import org.objenesis.instantiator.ObjectInstantiator; @Slf4j public class ProxyFactory { private final Objenesis objenesis = setupObjenesis(); private final TypeCache> byteBuddyCache = setupByteBuddyCache(); private static boolean hasDefaultConstructor(Class clazz) { try { clazz.getConstructor(); return true; } catch (NoSuchMethodException e) { return false; } } private TypeCache> setupByteBuddyCache() { try { return new TypeCache<>(Sort.WEAK); } catch (NoClassDefFoundError error) { log.info( "ByteBuddy is not on the classpath, so only interfaces can be used with transaction-outbox"); return null; } } private ObjenesisStd setupObjenesis() { try { return new ObjenesisStd(); } catch (NoClassDefFoundError error) { log.info( "Objenesis is not on the classpath, so only interfaces or classes with default constructors can be used with transaction-outbox"); return null; } } @SuppressWarnings({"unchecked", "cast"}) public T createProxy(Class clazz, BiFunction processor) { if (clazz.isInterface()) { // Fastest - we can just proxy an interface directly return (T) Proxy.newProxyInstance( clazz.getClassLoader(), new Class[] {clazz}, (proxy, method, args) -> processor.apply(method, args)); } else { Class proxy = buildByteBuddyProxyClass(clazz); return constructProxy(clazz, processor, proxy); } } private T constructProxy( Class clazz, BiFunction processor, Class proxy) { final T instance; if (hasDefaultConstructor(clazz)) { instance = Utils.uncheckedly(() -> proxy.getDeclaredConstructor().newInstance()); } else { if (objenesis == null) { throw new MissingOptionalDependencyException("org.objenesis", "objenesis"); } ObjectInstantiator instantiator = objenesis.getInstantiatorOf(proxy); instance = instantiator.newInstance(); } Utils.uncheck( () -> { var field = instance.getClass().getDeclaredField("handler"); field.set( instance, (InvocationHandler) (proxy1, method, args) -> processor.apply(method, args)); }); return instance; } @SuppressWarnings({"unchecked", "cast"}) private Class buildByteBuddyProxyClass(Class clazz) { if (byteBuddyCache == null) { throw new MissingOptionalDependencyException("net.bytebuddy", "byte-buddy"); } return (Class) byteBuddyCache.findOrInsert( clazz.getClassLoader(), clazz, () -> new ByteBuddy() .subclass(clazz) .defineField("handler", InvocationHandler.class, Visibility.PUBLIC) .method(ElementMatchers.isDeclaredBy(clazz)) .intercept(InvocationHandlerAdapter.toField("handler")) .make() .load(clazz.getClassLoader(), ClassLoadingStrategy.Default.INJECTION) .getLoaded()); } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/spi/SimpleTransaction.java ================================================ package com.gruelbox.transactionoutbox.spi; import static com.gruelbox.transactionoutbox.spi.Utils.safelyClose; import static com.gruelbox.transactionoutbox.spi.Utils.uncheck; import com.gruelbox.transactionoutbox.Transaction; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j @AllArgsConstructor(access = AccessLevel.PUBLIC) public final class SimpleTransaction implements Transaction, AutoCloseable { private final List postCommitHooks = new ArrayList<>(); private final Map preparedStatements = new HashMap<>(); private final Connection connection; private final Object context; @Override public Connection connection() { return connection; } @Override public void addPostCommitHook(Runnable runnable) { postCommitHooks.add(runnable); } @Override public PreparedStatement prepareBatchStatement(String sql) { return preparedStatements.computeIfAbsent( sql, s -> Utils.uncheckedly(() -> connection.prepareStatement(s))); } public void flushBatches() { if (!preparedStatements.isEmpty()) { log.debug("Flushing batches"); for (PreparedStatement statement : preparedStatements.values()) { uncheck(statement::executeBatch); } } } public void processHooks() { if (!postCommitHooks.isEmpty()) { log.debug("Running post-commit hooks"); postCommitHooks.forEach(Runnable::run); } } public void commit() { uncheck(connection::commit); } public void rollback() throws SQLException { connection.rollback(); } @SuppressWarnings("unchecked") @Override public T context() { return (T) context; } @Override public void close() { if (!preparedStatements.isEmpty()) { log.debug("Closing batch statements"); safelyClose(preparedStatements.values()); } } } ================================================ FILE: transactionoutbox-core/src/main/java/com/gruelbox/transactionoutbox/spi/Utils.java ================================================ package com.gruelbox.transactionoutbox.spi; import static java.util.stream.Collectors.joining; import com.gruelbox.transactionoutbox.ThrowingRunnable; import com.gruelbox.transactionoutbox.UncheckedException; import java.util.Arrays; import java.util.concurrent.Callable; import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; import org.slf4j.event.Level; @Slf4j public class Utils { @SuppressWarnings({"SameParameterValue", "WeakerAccess", "UnusedReturnValue"}) public static boolean safelyRun(String gerund, ThrowingRunnable runnable) { try { runnable.run(); return true; } catch (Exception e) { log.error("Error when {}", gerund, e); return false; } } @SuppressWarnings("unused") public static void safelyClose(AutoCloseable... closeables) { safelyClose(Arrays.asList(closeables)); } public static void safelyClose(Iterable closeables) { closeables.forEach( d -> { if (d == null) return; safelyRun("closing resource", d::close); }); } public static void uncheck(ThrowingRunnable runnable) { try { runnable.run(); } catch (Exception e) { uncheckAndThrow(e); } } public static T uncheckedly(Callable runnable) { try { return runnable.call(); } catch (Exception e) { return uncheckAndThrow(e); } } public static T uncheckAndThrow(Throwable e) { if (e instanceof RuntimeException) { throw (RuntimeException) e; } if (e instanceof Error) { throw (Error) e; } throw new UncheckedException(e); } public static T createLoggingProxy(ProxyFactory proxyFactory, Class clazz) { return proxyFactory.createProxy( clazz, (method, args) -> { log.info( "Called mock " + clazz.getSimpleName() + ".{}({})", method.getName(), args == null ? "" : Arrays.stream(args) .map(it -> it == null ? "null" : it.toString()) .collect(Collectors.joining(", "))); return null; }); } public static T firstNonNull(T one, Supplier two) { if (one == null) return two.get(); return one; } public static void logAtLevel(Logger logger, Level level, String message, Object... args) { switch (level) { case ERROR: logger.error(message, args); break; case WARN: logger.warn(message, args); break; case INFO: logger.info(message, args); break; case DEBUG: logger.debug(message, args); break; case TRACE: logger.trace(message, args); break; default: logger.warn(message, args); break; } } public static String stringify(Object o) { if (o == null) { return "null"; } if (o.getClass().isArray()) { return "[" + Arrays.stream((Object[]) o).map(Utils::stringify).collect(joining(", ")) + "]"; } if (o instanceof String) { return "\"" + o + "\""; } return o.toString(); } } ================================================ FILE: transactionoutbox-core/src/test/java/com/gruelbox/transactionoutbox/AbstractTestDefaultInvocationSerializer.java ================================================ package com.gruelbox.transactionoutbox; import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; import java.io.UncheckedIOException; import java.time.*; import java.time.temporal.ChronoUnit; import java.util.*; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @SuppressWarnings("RedundantCast") @Slf4j abstract class AbstractTestDefaultInvocationSerializer { protected static final String CLASS_NAME = "foo"; protected static final String METHOD_NAME = "bar"; private final DefaultInvocationSerializer serializer; protected AbstractTestDefaultInvocationSerializer(Integer version) { this.serializer = DefaultInvocationSerializer.builder() .serializableTypes(Set.of(ExampleCustomEnum.class, ExampleCustomClass.class)) .version(version) .build(); } @Test void testNoArgs() { check(new Invocation(String.class.getName(), "toString", new Class[0], new Object[0])); } @Test void testArrays() { check( new Invocation( CLASS_NAME, METHOD_NAME, new Class[] {int[].class}, new Object[] {new int[] {1, 2, 3}})); check( new Invocation( CLASS_NAME, METHOD_NAME, new Class[] {Integer[].class}, new Object[] {new Integer[] {1, 2, 3}})); check( new Invocation( CLASS_NAME, METHOD_NAME, new Class[] {String[].class}, new Object[] {new String[] {"1", "2", "3"}})); } @Test void testPrimitives() { Class[] primitives = { byte.class, short.class, int.class, long.class, float.class, double.class, boolean.class, char.class }; Object[] values = {(byte) 1, (short) 2, 3, 4L, 1.23F, 1.23D, true, '-'}; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testBoxedPrimitives() { Class[] primitives = { Byte.class, Short.class, Integer.class, Long.class, Float.class, Double.class, Boolean.class, Character.class, String.class }; Object[] values = { (Byte) (byte) 1, (Short) (short) 2, (Integer) 3, (Long) 4L, (Float) 1.23F, (Double) 1.23D, (Boolean) true, (Character) '-', "Foo" }; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testJavaDateEnums() { Class[] primitives = {DayOfWeek.class, Month.class, ChronoUnit.class}; Object[] values = {DayOfWeek.FRIDAY, Month.APRIL, ChronoUnit.DAYS}; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testJavaDateEnumsNulls() { Class[] primitives = {DayOfWeek.class, Month.class, ChronoUnit.class}; Object[] values = {null, null, null}; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testJavaUtilDate() { Class[] primitives = {Date.class, Date.class}; Object[] values = {new Date(), null}; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testJavaTimeClasses() { Class[] primitives = { Duration.class, Instant.class, LocalDate.class, LocalDateTime.class, MonthDay.class, Period.class, Year.class, YearMonth.class, ZonedDateTime.class }; Object[] values = { Duration.ofDays(1), Instant.now(), LocalDate.now(), LocalDateTime.now(), MonthDay.of(1, 1), Period.ofMonths(1), Year.now(), YearMonth.now(), ZonedDateTime.now() }; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testJavaTimeClassesNulls() { Class[] primitives = { Duration.class, Instant.class, LocalDate.class, LocalDateTime.class, MonthDay.class, Period.class, Year.class, YearMonth.class, ZonedDateTime.class }; Object[] values = new Object[9]; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testCustomEnum() { Class[] primitives = {ExampleCustomEnum.class, ExampleCustomEnum.class}; Object[] values = {ExampleCustomEnum.ONE, ExampleCustomEnum.TWO}; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testCustomEnumNulls() { Class[] primitives = {ExampleCustomEnum.class}; Object[] values = {null}; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testCustomComplexClass() { Class[] primitives = {ExampleCustomClass.class, ExampleCustomClass.class}; Object[] values = { new ExampleCustomClass("Foo", "Bar"), new ExampleCustomClass("Bish", "Bash") }; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testMDCAndSession() { Class[] primitives = {Integer.class}; Object[] values = {1}; check( new Invocation( CLASS_NAME, METHOD_NAME, primitives, values, Map.of("A", "1", "B", "2"), Map.of("C", "3"))); } @Test void testSession() { Class[] primitives = {Integer.class}; Object[] values = {1}; check( new Invocation( CLASS_NAME, METHOD_NAME, primitives, values, null, Map.of("A", "1", "B", "2"))); } @Test void testUUID() { Class[] primitives = {UUID.class}; Object[] values = {UUID.randomUUID()}; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testUUIDNull() { Class[] primitives = {UUID.class}; Object[] values = {null}; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testDeserializationException() { assertThrows( IOException.class, () -> serializer.deserializeInvocation(new StringReader("unparseable"))); } void check(Invocation invocation) { Invocation deserialized = serdeser(invocation); Assertions.assertEquals(deserialized, serdeser(invocation)); Assertions.assertEquals(invocation, deserialized); } Invocation serdeser(Invocation invocation) { try { var writer = new StringWriter(); serializer.serializeInvocation(invocation, writer); log.info("Serialised as: {}", writer); return serializer.deserializeInvocation(new StringReader(writer.toString())); } catch (IOException e) { throw new UncheckedIOException(e); } } enum ExampleCustomEnum { ONE, TWO } @Getter static class ExampleCustomClass { private final String arg1; private final String arg2; ExampleCustomClass(String arg1, String arg2) { this.arg1 = arg1; this.arg2 = arg2; } @Override public String toString() { return "ExampleCustomClass{" + "arg1='" + arg1 + '\'' + ", arg2='" + arg2 + '\'' + '}'; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ExampleCustomClass that = (ExampleCustomClass) o; return Objects.equals(arg1, that.arg1) && Objects.equals(arg2, that.arg2); } @Override public int hashCode() { return Objects.hash(arg1, arg2); } } } ================================================ FILE: transactionoutbox-core/src/test/java/com/gruelbox/transactionoutbox/TestDefaultInvocationSerializer.java ================================================ package com.gruelbox.transactionoutbox; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Nested; @Slf4j class TestDefaultInvocationSerializer { @Nested class Version1 extends AbstractTestDefaultInvocationSerializer { public Version1() { super(1); } @Override void testSession() { // no-op } @Override void testMDCAndSession() { // no-op } } @Nested class Version2 extends AbstractTestDefaultInvocationSerializer { public Version2() { super(2); } } @Nested class NullVersion extends AbstractTestDefaultInvocationSerializer { public NullVersion() { super(null); } } } ================================================ FILE: transactionoutbox-core/src/test/java/com/gruelbox/transactionoutbox/TestDefaultMigrationManager.java ================================================ package com.gruelbox.transactionoutbox; import static com.gruelbox.transactionoutbox.Dialect.H2; import static org.junit.jupiter.api.Assertions.fail; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import java.util.concurrent.*; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @Slf4j public class TestDefaultMigrationManager { private static HikariDataSource dataSource; @BeforeAll static void beforeAll() { HikariConfig config = new HikariConfig(); config.setJdbcUrl( "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DEFAULT_LOCK_TIMEOUT=60000;LOB_TIMEOUT=2000;MV_STORE=TRUE;DATABASE_TO_UPPER=FALSE"); config.setUsername("test"); config.setPassword("test"); config.addDataSourceProperty("cachePrepStmts", "true"); dataSource = new HikariDataSource(config); } @AfterAll static void afterAll() { dataSource.close(); } @Test void parallelMigrations() { CountDownLatch readyLatch = new CountDownLatch(2); DefaultMigrationManager.withLatch( readyLatch, waitLatch -> { Executor executor = runnable -> new Thread(runnable).start(); TransactionManager txm = TransactionManager.fromDataSource(dataSource); CompletableFuture threads = CompletableFuture.allOf( CompletableFuture.runAsync( () -> { try { DefaultMigrationManager.migrate(txm, H2); } catch (Exception e) { log.error("Thread 1 failed", e); throw e; } }, executor), CompletableFuture.runAsync( () -> { try { DefaultMigrationManager.migrate(txm, H2); } catch (Exception e) { log.error("Thread 2 failed", e); throw e; } }, executor)); try { if (!readyLatch.await(15, TimeUnit.SECONDS)) { throw new TimeoutException(); } waitLatch.countDown(); } catch (InterruptedException | TimeoutException e) { fail("Timed out or interrupted waiting for ready latch"); } finally { threads.join(); } }); } } ================================================ FILE: transactionoutbox-core/src/test/java/com/gruelbox/transactionoutbox/TestDefaultPersistorConfiguration.java ================================================ package com.gruelbox.transactionoutbox; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; import java.io.StringWriter; import java.sql.ResultSet; import java.sql.Statement; import org.junit.jupiter.api.Test; class TestDefaultPersistorConfiguration { @Test final void whenMigrateIsFalseDoNotMigrate() throws Exception { TransactionManager transactionManager = simpleTxnManager(); runSql(transactionManager, "DROP ALL OBJECTS"); TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(DefaultPersistor.builder().dialect(Dialect.H2).migrate(false).build()) .build(); transactionManager.inTransactionThrows( tx -> { try (Statement statement = tx.connection().createStatement()) { try (ResultSet rs = statement.executeQuery( "SELECT COUNT(*)" + " FROM INFORMATION_SCHEMA.TABLES" + " WHERE TABLE_NAME IN ('TXNO_OUTBOX', 'TXNO_VERSION')")) { rs.next(); assertThat(rs.getInt(1), is(0)); } } }); } @Test final void writeSchema() { StringWriter stringWriter = new StringWriter(); DefaultPersistor defaultPersistor = DefaultPersistor.builder().dialect(Dialect.H2).build(); defaultPersistor.writeSchema(stringWriter); String migrations = stringWriter.toString(); assertThat(migrations, startsWith("-- 1: Create outbox table")); assertThat( migrations, containsString( "-- 2: Add unique request id" + System.lineSeparator() + "ALTER TABLE TXNO_OUTBOX ADD COLUMN uniqueRequestId VARCHAR(100) NULL UNIQUE")); assertThat( migrations, containsString( "-- 8: Update length of invocation column on outbox for MySQL dialects only." + System.lineSeparator() + "-- Nothing for H2")); } private TransactionManager simpleTxnManager() { return TransactionManager.fromConnectionDetails( "org.h2.Driver", "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DEFAULT_LOCK_TIMEOUT=60000;LOB_TIMEOUT=2000;MV_STORE=TRUE", "test", "test"); } private void runSql( TransactionManager transactionManager, @SuppressWarnings("SameParameterValue") String sql) { transactionManager.inTransaction( tx -> { try { try (Statement statement = tx.connection().createStatement()) { statement.execute(sql); } } catch (Exception e) { throw new RuntimeException(e); } }); } } ================================================ FILE: transactionoutbox-core/src/test/java/com/gruelbox/transactionoutbox/TestProxyGeneration.java ================================================ package com.gruelbox.transactionoutbox; import static org.junit.jupiter.api.Assertions.assertTrue; import com.gruelbox.transactionoutbox.spi.ProxyFactory; import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class TestProxyGeneration { private ProxyFactory proxyFactory; @BeforeEach void setUp() { proxyFactory = new ProxyFactory(); } /** Reflection */ @Test void testReflection() { AtomicBoolean called = new AtomicBoolean(); Interface proxy = proxyFactory.createProxy( Interface.class, (method, args) -> { called.set(true); return null; }); proxy.doThing(); assertTrue(called.get()); } /** ByteBuddy */ @Test void testByteBuddy() { AtomicBoolean called = new AtomicBoolean(); Child proxy = proxyFactory.createProxy( Child.class, (method, args) -> { called.set(true); return null; }); proxy.doThing(); assertTrue(called.get()); } /** This fails without Objenesis. */ @Test void testObjensis() { AtomicBoolean called = new AtomicBoolean(); Parent proxy = proxyFactory.createProxy( Parent.class, (method, args) -> { called.set(true); return null; }); proxy.doThing(); assertTrue(called.get()); } interface Interface { void doThing(); } static class Child { void doThing() { // No-op } } static class Parent { private final Child child; Parent(Child child) { this.child = child; } void doThing() { // No-op } } } ================================================ FILE: transactionoutbox-core/src/test/java/com/gruelbox/transactionoutbox/TestValidator.java ================================================ package com.gruelbox.transactionoutbox; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import java.math.BigDecimal; import java.time.Clock; import java.time.Instant; import java.time.ZoneId; import org.junit.jupiter.api.Test; class TestValidator { private static final Invocation COMPLEX_INVOCATION = new Invocation( "Foo", "Bar", new Class[] {int.class, BigDecimal.class, String.class}, new Object[] {1, BigDecimal.TEN, null}); private final Instant now = Instant.now(); private final Validator validator = new Validator(() -> Clock.fixed(now, ZoneId.of("+4"))); @Test void testEntryDateFuture() { TransactionOutboxEntry entry = TransactionOutboxEntry.builder() .id("FOO") .invocation(COMPLEX_INVOCATION) .nextAttemptTime(now.plusMillis(1)) .build(); assertDoesNotThrow(() -> validator.validate(entry)); } } ================================================ FILE: transactionoutbox-core/src/test/resources/logback-test.xml ================================================ %d{HH:mm:ss.SSS} [%thread] %-5level SESSION-KEY=%X{SESSION-KEY} %logger{5} - %msg%n ================================================ FILE: transactionoutbox-guice/README.md ================================================ # transaction-outbox-guice [![Guice on Maven Central](https://maven-badges.herokuapp.com/maven-central/com.gruelbox/transactionoutbox-guice/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.gruelbox/transactionoutbox-guice) [![Guice Javadoc](https://www.javadoc.io/badge/com.gruelbox/transactionoutbox-guice.svg?color=blue)](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-guice) [![Latest snapshot](https://img.shields.io/github/v/tag/gruelbox/transaction-outbox?label=snapshot&sort=semver)](#development-snapshots) Extension for [transaction-outbox-core](../README.md) which integrates with Guice. ## Installation ### Stable releases #### Maven ```xml com.gruelbox transactionoutbox-guice 7.0.707 ``` #### Gradle ```groovy implementation 'com.gruelbox:transactionoutbox-guice:7.0.707' ``` ### Development snapshots See [transactionoutbox-core](../README.md) for more information. ## Standard usage ### Configuration To get a `TransactionOutbox` for use throughout your application, add a `Singleton` binding for your chosen transaction manager and then wire in `GuiceInstantiator` as follows: ```java @Provides @Singleton TransactionOutbox transactionOutbox(Injector injector, TransactionManager transactionManager) { return TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(Dialect.MY_SQL_8)) .instantiator(GuiceInstantiator.builder().injector(injector).build()) .build(); } ``` ### Usage ```java @Inject private TransactionOutbox outbox; void doSomething() { // Obtains a MyService from the injector outbox.schedule(MyService.class).doAThing(1, 2, 3); } ``` ## Remote injection Alternatively, you may prefer to hide the use of `TransactionOutbox` and create injectable "remote" implementations of specific services. This is a stylistic choice, and is more a Guice thing than a `TransactionOutbox` thing, but is presented here for illustration. ### Configuration Create a suitable binding annotation to specify that you want to inject the remote version of a service: ```java @Target({ElementType.PARAMETER, ElementType.METHOD, ElementType.FIELD}) @Retention(RetentionPolicy.RUNTIME) @BindingAnnotation public @interface Remote {} ``` Bind `TransactionOutbox` as per the example above, and add two more bindings to expose the "real" and "remote" versions of the service: ```java @Provides @Remote @Singleton // Can help performance MyService remote(TransactionOutbox outbox) { return outbox.schedule(MyService.class); } @Provides MyService local() { return new MyService(); } ``` ### Usage Now you can inject the remote implementation and use it to schedule work. The following is exactly equivalent to the usage example above, just using an injected remote: ```java @Inject @Remote private MyService myService; void doSomething() { myService.doAThing(1, 2, 3); } ``` ================================================ FILE: transactionoutbox-guice/pom.xml ================================================ transactionoutbox-parent com.gruelbox ${revision} 4.0.0 Transaction Outbox Guice jar transactionoutbox-guice A safe implementation of the transactional outbox pattern for Java (Guice extension library) 5.2.4.RELEASE com.gruelbox transactionoutbox-core ${project.version} com.google.inject guice 7.0.0 com.google.guava guava com.google.guava guava 33.5.0-jre test org.projectlombok lombok ch.qos.logback logback-classic ch.qos.logback logback-core org.hamcrest hamcrest-core org.junit.jupiter junit-jupiter-engine org.mockito mockito-all ================================================ FILE: transactionoutbox-guice/src/main/java/com/gruelbox/transactionoutbox/guice/GuiceInstantiator.java ================================================ package com.gruelbox.transactionoutbox.guice; import com.google.inject.Injector; import com.gruelbox.transactionoutbox.spi.AbstractFullyQualifiedNameInstantiator; import lombok.experimental.SuperBuilder; /** Instantiator that uses the Guice {@link Injector} to source objects. */ @SuperBuilder public class GuiceInstantiator extends AbstractFullyQualifiedNameInstantiator { private final Injector injector; @Override protected Object createInstance(Class clazz) { return injector.getInstance(clazz); } } ================================================ FILE: transactionoutbox-guice/src/test/java/com/gruelbox/transactionoutbox/guice/acceptance/TestGuiceBinding.java ================================================ package com.gruelbox.transactionoutbox.guice.acceptance; import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.inject.AbstractModule; import com.google.inject.BindingAnnotation; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Provides; import com.google.inject.Singleton; import com.gruelbox.transactionoutbox.StubPersistor; import com.gruelbox.transactionoutbox.StubThreadLocalTransactionManager; import com.gruelbox.transactionoutbox.Submitter; import com.gruelbox.transactionoutbox.TransactionManager; import com.gruelbox.transactionoutbox.TransactionOutbox; import com.gruelbox.transactionoutbox.TransactionOutboxEntry; import com.gruelbox.transactionoutbox.TransactionOutboxListener; import com.gruelbox.transactionoutbox.guice.GuiceInstantiator; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.util.concurrent.atomic.AtomicBoolean; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; /** * Demonstrates an alternative approach to using {@link TransactionOutbox} using binding annotations * to inject a remoted version of a service. */ @Slf4j class TestGuiceBinding { /** The real service */ @Inject MyService local; /** The remoted version */ @Inject @Remote MyService remote; /** We need this to schedule the work */ @Inject TransactionManager transactionManager; @Test void testProviderInjection() { AtomicBoolean processedWithRemote = new AtomicBoolean(); Injector injector = Guice.createInjector(new DemoModule(processedWithRemote)); injector.injectMembers(this); transactionManager.inTransaction( () -> { remote.process(); log.info("Queued request"); }); assertTrue(processedWithRemote.get()); assertTrue(local.processed.get()); } /** The service we're going to remote */ static class MyService { AtomicBoolean processed = new AtomicBoolean(); void process() { processed.set(true); log.info("Processed local"); } } /** Binding annotation for the remote version of the service. */ @Target({ElementType.PARAMETER, ElementType.METHOD, ElementType.FIELD}) @Retention(RetentionPolicy.RUNTIME) @BindingAnnotation public @interface Remote {} /** Sets up the bindings */ static final class DemoModule extends AbstractModule { private final AtomicBoolean processedWithRemote; DemoModule(AtomicBoolean processedWithRemote) { this.processedWithRemote = processedWithRemote; } @Provides @Singleton TransactionManager manager() { return new StubThreadLocalTransactionManager(); } @Provides @Singleton TransactionOutbox outbox(Injector injector, TransactionManager transactionManager) { return TransactionOutbox.builder() .instantiator(GuiceInstantiator.builder().injector(injector).build()) .persistor(StubPersistor.builder().build()) .submitter(Submitter.withExecutor(Runnable::run)) .transactionManager(transactionManager) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { processedWithRemote.set(true); } }) .build(); } @Provides @Remote @Singleton MyService remote(TransactionOutbox outbox) { return outbox.schedule(MyService.class); } @Provides @Singleton MyService local() { return new MyService(); } } } ================================================ FILE: transactionoutbox-guice/src/test/java/com/gruelbox/transactionoutbox/guice/acceptance/TestGuiceInstantiator.java ================================================ package com.gruelbox.transactionoutbox.guice.acceptance; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import com.gruelbox.transactionoutbox.guice.GuiceInstantiator; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.junit.jupiter.api.Test; class TestGuiceInstantiator { @Test void testInjection() { Injector injector = Guice.createInjector(); GuiceInstantiator guiceInstantiator = GuiceInstantiator.builder().injector(injector).build(); Object instance = guiceInstantiator.getInstance(Parent.class.getName()); MatcherAssert.assertThat(instance, Matchers.isA(Parent.class)); } static final class Child {} static final class Parent { private final Child child; @Inject Parent(Child child) { this.child = child; } } } ================================================ FILE: transactionoutbox-guice/src/test/resources/logback-test.xml ================================================ %d{HH:mm:ss.SSS} [%thread] %-5level SESSION-KEY=%X{SESSION-KEY} %logger{5} - %msg%n ================================================ FILE: transactionoutbox-jackson/README.md ================================================ # transaction-outbox-jackson [![Jackson on Maven Central](https://maven-badges.herokuapp.com/maven-central/com.gruelbox/transactionoutbox-jackson/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.gruelbox/transactionoutbox-guice) [![Jackson Javadoc](https://www.javadoc.io/badge/com.gruelbox/transactionoutbox-jackson.svg?color=blue)](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-guice) [![Latest snapshot](https://img.shields.io/github/v/tag/gruelbox/transaction-outbox?label=snapshot&sort=semver)](#development-snapshots) Extension for [transaction-outbox-core](../README.md) which uses Jackson for serialisation. If you are confident in trusting your database, then this serializer has a number of advantages: it is as configurable as whatever Jackson's `ObjectMapper` can handle, and explicitly handles n-depth polymorphic trees. This largely means that you can throw pretty much anything at it and it will "just work". However, if there is any risk that you might not trust the source of the serialized `Invocation`, _do not use this_. This serializer is vulnerable to [deserialization of untrusted data](https://github.com/gruelbox/transaction-outbox/issues/236#issuecomment-1024929436), which is why it is not included in the core library. ## Installation ### Stable releases #### Maven ```xml com.gruelbox transactionoutbox-jackson 7.0.707 ``` #### Gradle ```groovy implementation 'com.gruelbox:transactionoutbox-jackson:7.0.707' ``` ### Development snapshots See [transactionoutbox-core](../README.md) for more information. ## Configuration ### Fresh projects If starting with a fresh project, you don't need to worry about compatibility with DefaultInvocationSerializer, so configure as follows: ```java var outbox = TransactionOutbox.builder() .persistor(DefaultPersistor.builder() .dialect(Dialect.H2) .serializer(JacksonInvocationSerializer.builder() .mapper(new ObjectMapper()) .build()) .build()) ``` ### Existing projects using DefaultInvocationSerializer If you're already using Transaction Outbox, you may have outbox tasks queued which your application needs to continue to be capable of loading. To handle this, pass through an instance of `DefaultInvocationSerializer` which matches what you used previously: ```java var outbox = TransactionOutbox.builder() .persistor(DefaultPersistor.builder() .dialect(Dialect.H2) .serializer(JacksonInvocationSerializer.builder() .mapper(new ObjectMapper()) .defaultInvocationSerializer(DefaultInvocationSerializer.builder() .serializableTypes(Set.of(Foo.class, Bar.class)) .build()) .build()) .build()) ``` ## Usage You can now go wild with your scheduled method arguments: ```java outbox.schedule(getClass()) .process(List.of(LocalDate.of(2000,1,1), "a", "b", 2)); ``` ================================================ FILE: transactionoutbox-jackson/pom.xml ================================================ transactionoutbox-parent com.gruelbox ${revision} 4.0.0 Transaction Outbox Jackson jar transactionoutbox-jackson A safe implementation of the transactional outbox pattern for Java (Jackson extension library) 2.21.0 3.20.0 com.gruelbox transactionoutbox-core ${project.version} com.fasterxml.jackson.core jackson-databind ${jackson.version} org.apache.commons commons-lang3 ${commons.lang.version} org.projectlombok lombok com.gruelbox transactionoutbox-testing ${project.version} test com.fasterxml.jackson.datatype jackson-datatype-guava ${jackson.version} test com.google.guava guava com.google.guava guava 33.5.0-jre test com.fasterxml.jackson.datatype jackson-datatype-jdk8 ${jackson.version} test com.fasterxml.jackson.datatype jackson-datatype-jsr310 ${jackson.version} test com.h2database h2 ================================================ FILE: transactionoutbox-jackson/src/main/java/com/gruelbox/transactionoutbox/jackson/CustomInvocationDeserializer.java ================================================ package com.gruelbox.transactionoutbox.jackson; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.fasterxml.jackson.databind.jsontype.TypeDeserializer; import com.fasterxml.jackson.databind.node.ArrayNode; import com.gruelbox.transactionoutbox.Invocation; import java.io.IOException; import java.util.Map; import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ClassUtils; @Slf4j class CustomInvocationDeserializer extends StdDeserializer { private static final Pattern setPattern = Pattern.compile("\\{\\w*\"(java.util.ImmutableCollections\\$Set[\\dN]+)\"\\w*:"); private static final Pattern mapPattern = Pattern.compile("\\{\\w*\"(java.util.ImmutableCollections\\$Map[\\dN]+)\"\\w*:"); private static final Pattern listPattern = Pattern.compile("\\{\\w*\"(java.util.ImmutableCollections\\$List[\\dN]+)\"\\w*:"); protected CustomInvocationDeserializer(Class vc) { super(vc); } CustomInvocationDeserializer() { this(Invocation.class); } @Override public Invocation deserializeWithType( JsonParser p, DeserializationContext ctxt, TypeDeserializer typeDeserializer) throws IOException { return deserialize(p, ctxt); } @Override public Invocation deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { JsonNode node = p.getCodec().readTree(p); String className = node.get("className").textValue(); String methodName = node.get("methodName").textValue(); ArrayNode paramTypes = ((ArrayNode) node.get("parameterTypes")); JsonNode arguments = node.get("args"); JsonNode processedArguments = replaceImmutableCollections(arguments, p); Class[] types = new Class[paramTypes.size()]; for (int i = 0; i < paramTypes.size(); i++) { try { types[i] = ClassUtils.getClass(paramTypes.get(i).asText()); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } Object[] args = p.getCodec().treeToValue(processedArguments, Object[].class); Map mdc = p.getCodec() .readValue(p.getCodec().treeAsTokens(node.get("mdc")), new TypeReference<>() {}); var sessionNode = node.get("session"); Map session = null; if (sessionNode != null && !sessionNode.isNull()) { session = p.getCodec().readValue(p.getCodec().treeAsTokens(sessionNode), new TypeReference<>() {}); } return new Invocation(className, methodName, types, args, mdc, session); } private JsonNode replaceImmutableCollections(JsonNode arguments, JsonParser p) throws IOException { String args = arguments.toString(); args = setPattern.matcher(args).replaceAll("{\"java.util.HashSet\":"); args = mapPattern.matcher(args).replaceAll("{\"java.util.HashMap\":"); args = listPattern.matcher(args).replaceAll("{\"java.util.ArrayList\":"); JsonParser parser = p.getCodec().getFactory().createParser(args); return p.getCodec().readTree(parser); } } ================================================ FILE: transactionoutbox-jackson/src/main/java/com/gruelbox/transactionoutbox/jackson/CustomInvocationSerializer.java ================================================ package com.gruelbox.transactionoutbox.jackson; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.jsontype.TypeSerializer; import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.gruelbox.transactionoutbox.Invocation; import java.io.IOException; class CustomInvocationSerializer extends StdSerializer { public CustomInvocationSerializer() { this(Invocation.class); } protected CustomInvocationSerializer(Class t) { super(t); } @Override public void serializeWithType( Invocation value, JsonGenerator gen, SerializerProvider serializers, TypeSerializer typeSer) throws IOException { serialize(value, gen, serializers); } @Override public void serialize(Invocation value, JsonGenerator gen, SerializerProvider provider) throws IOException { gen.writeStartObject(); gen.writeStringField("className", value.getClassName()); gen.writeStringField("methodName", value.getMethodName()); gen.writeArrayFieldStart("parameterTypes"); for (Class parameterType : value.getParameterTypes()) { gen.writeString(parameterType.getCanonicalName()); } gen.writeEndArray(); gen.writeObjectField("args", value.getArgs()); gen.writeObjectField("mdc", value.getMdc()); gen.writeObjectField("session", value.getSession()); gen.writeEndObject(); } } ================================================ FILE: transactionoutbox-jackson/src/main/java/com/gruelbox/transactionoutbox/jackson/JacksonInvocationSerializer.java ================================================ package com.gruelbox.transactionoutbox.jackson; import com.fasterxml.jackson.databind.ObjectMapper; import com.gruelbox.transactionoutbox.DefaultInvocationSerializer; import com.gruelbox.transactionoutbox.Invocation; import com.gruelbox.transactionoutbox.InvocationSerializer; import java.io.BufferedReader; import java.io.IOException; import java.io.Reader; import java.io.Writer; import lombok.Builder; /** * A general-purpose {@link InvocationSerializer} which can handle pretty much anything that you * throw at it. * *

Note that if there is any risk that you might not trust the source of the serialized {@link * Invocation}, do not use this. This serializer is vulnerable to a * deserialization of untrusted data vulnerability (more information here) * which is why it is not included in the core library. */ public final class JacksonInvocationSerializer implements InvocationSerializer { private final ObjectMapper mapper; private final InvocationSerializer defaultInvocationSerializer; @Builder private JacksonInvocationSerializer( ObjectMapper mapper, DefaultInvocationSerializer defaultInvocationSerializer) { this.mapper = mapper.copy(); this.defaultInvocationSerializer = defaultInvocationSerializer; this.mapper.setDefaultTyping(TransactionOutboxJacksonModule.typeResolver()); this.mapper.registerModule(new TransactionOutboxJacksonModule()); } @Override public void serializeInvocation(Invocation invocation, Writer writer) { try { mapper.writeValue(writer, invocation); } catch (IOException e) { throw new RuntimeException(e); } } @Override public Invocation deserializeInvocation(Reader reader) throws IOException { // read ahead to check if old style BufferedReader br = new BufferedReader(reader); if (checkForOldSerialization(br)) { if (defaultInvocationSerializer == null) { throw new UnsupportedOperationException( "Can't deserialize GSON-format tasks without a " + DefaultInvocationSerializer.class.getSimpleName() + ". Supply one when building " + getClass().getSimpleName()); } return defaultInvocationSerializer.deserializeInvocation(br); } return mapper.readValue(br, Invocation.class); } private boolean checkForOldSerialization(BufferedReader reader) throws IOException { reader.mark(1); char[] chars = new char[6]; int charsRead = reader.read(chars, 0, 6); String result = ""; if (charsRead != -1) { result = new String(chars, 0, charsRead); } reader.reset(); return result.startsWith("{\"c\":"); } } ================================================ FILE: transactionoutbox-jackson/src/main/java/com/gruelbox/transactionoutbox/jackson/TransactionOutboxEntryDeserializer.java ================================================ package com.gruelbox.transactionoutbox.jackson; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.ObjectCodec; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; import com.gruelbox.transactionoutbox.Invocation; import com.gruelbox.transactionoutbox.TransactionOutboxEntry; import java.io.IOException; import java.time.Instant; import java.util.Map; class TransactionOutboxEntryDeserializer extends JsonDeserializer { @Override public TransactionOutboxEntry deserialize(JsonParser p, DeserializationContext c) throws IOException { ObjectCodec oc = p.getCodec(); JsonNode entry = oc.readTree(p); var invocation = entry.get("invocation"); return TransactionOutboxEntry.builder() .id(entry.get("id").asText()) .lastAttemptTime(mapNullableInstant(entry.get("lastAttemptTime"), c)) .nextAttemptTime(mapNullableInstant(entry.get("nextAttemptTime"), c)) .attempts(entry.get("attempts").asInt()) .blocked(entry.get("blocked").asBoolean()) .processed(entry.get("processed").asBoolean()) .uniqueRequestId(mapNullableString(entry.get("uniqueRequestId"))) .version(entry.get("version").asInt()) .invocation( new Invocation( invocation.get("className").asText(), invocation.get("methodName").asText(), c.readTreeAsValue(invocation.get("parameterTypes"), Class[].class), c.readTreeAsValue(invocation.get("args"), Object[].class), mapNullableStringMap(invocation.get("mdc"), c), mapNullableStringMap(invocation.get("session"), c))) .build(); } private String mapNullableString(JsonNode node) { if (node == null || node.isNull()) { return null; } return node.asText(); } private Instant mapNullableInstant(JsonNode node, DeserializationContext c) throws IOException { if (node == null || node.isNull()) { return null; } return c.readTreeAsValue(node, Instant.class); } private Map mapNullableStringMap(JsonNode node, DeserializationContext c) throws IOException { if (node == null || node.isNull()) { return null; } return c.readTreeAsValue( node, c.getTypeFactory().constructType(new TypeReference>() {})); } } ================================================ FILE: transactionoutbox-jackson/src/main/java/com/gruelbox/transactionoutbox/jackson/TransactionOutboxJacksonModule.java ================================================ package com.gruelbox.transactionoutbox.jackson; import static com.fasterxml.jackson.databind.ObjectMapper.DefaultTyping.NON_FINAL; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.jsontype.BasicPolymorphicTypeValidator; import com.fasterxml.jackson.databind.jsontype.TypeResolverBuilder; import com.fasterxml.jackson.databind.module.SimpleDeserializers; import com.fasterxml.jackson.databind.module.SimpleSerializers; import com.gruelbox.transactionoutbox.Invocation; import com.gruelbox.transactionoutbox.TransactionOutboxEntry; public class TransactionOutboxJacksonModule extends Module { @Override public String getModuleName() { return "TransactionOutboxJacksonModule"; } @Override public Version version() { return Version.unknownVersion(); } @Override public void setupModule(SetupContext setupContext) { SimpleSerializers serializers = new SimpleSerializers(); serializers.addSerializer(Invocation.class, new CustomInvocationSerializer()); setupContext.addSerializers(serializers); SimpleDeserializers deserializers = new SimpleDeserializers(); deserializers.addDeserializer(Invocation.class, new CustomInvocationDeserializer()); deserializers.addDeserializer( TransactionOutboxEntry.class, new TransactionOutboxEntryDeserializer()); setupContext.addDeserializers(deserializers); } public static TypeResolverBuilder typeResolver() { return new ObjectMapper.DefaultTypeResolverBuilder( NON_FINAL, BasicPolymorphicTypeValidator.builder().allowIfBaseType(Object.class).build()) .init(JsonTypeInfo.Id.CLASS, null) .inclusion(JsonTypeInfo.As.WRAPPER_OBJECT); } } ================================================ FILE: transactionoutbox-jackson/src/test/java/com/gruelbox/transactionoutbox/jackson/MonetaryAmount.java ================================================ package com.gruelbox.transactionoutbox.jackson; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.math.BigDecimal; import java.util.Objects; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @AllArgsConstructor @NoArgsConstructor @Getter @Setter @JsonIgnoreProperties(ignoreUnknown = true) public final class MonetaryAmount { private static final String GBP = "GBP"; private static final BigDecimal BY_HUNDRED = BigDecimal.valueOf(100); public static final MonetaryAmount ZERO_GBP = new MonetaryAmount(BigDecimal.ZERO, GBP); public static final MonetaryAmount TEN_GBP = new MonetaryAmount(BigDecimal.TEN, GBP); public static final MonetaryAmount ONE_HUNDRED_GBP = new MonetaryAmount(BigDecimal.valueOf(100), GBP); private BigDecimal amount; private String currency; public static MonetaryAmount ofGbp(final String amount) { return new MonetaryAmount(new BigDecimal(amount), "GBP"); } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; MonetaryAmount that = (MonetaryAmount) o; if (amount != null ? amount.compareTo(that.amount) != 0 : that.amount != null) return false; return Objects.equals(currency, that.currency); } @Override public int hashCode() { int result = amount != null ? amount.hashCode() : 0; result = 31 * result + (currency != null ? currency.hashCode() : 0); return result; } @Override public String toString() { return currency + " " + amount.stripTrailingZeros().toPlainString(); } } ================================================ FILE: transactionoutbox-jackson/src/test/java/com/gruelbox/transactionoutbox/jackson/SerializationStressTestInput.java ================================================ package com.gruelbox.transactionoutbox.jackson; import java.util.Map; import java.util.Set; import lombok.*; @Getter @Setter @NoArgsConstructor @EqualsAndHashCode @ToString public class SerializationStressTestInput { private boolean enabled = false; private MonetaryAmount amount = MonetaryAmount.ONE_HUNDRED_GBP; private Set investments = Set.of("investment1", "investment2", "investment3"); private Map investmentAmounts = Map.of( "investment1", MonetaryAmount.ofGbp("33"), "investment2", MonetaryAmount.ofGbp("34"), "investment3", MonetaryAmount.ofGbp("33")); } ================================================ FILE: transactionoutbox-jackson/src/test/java/com/gruelbox/transactionoutbox/jackson/TestJacksonInvocationSerializer.java ================================================ package com.gruelbox.transactionoutbox.jackson; import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.guava.GuavaModule; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.gruelbox.transactionoutbox.DefaultInvocationSerializer; import com.gruelbox.transactionoutbox.Invocation; import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; import java.io.UncheckedIOException; import java.time.*; import java.time.temporal.ChronoUnit; import java.util.*; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @SuppressWarnings("RedundantCast") class TestJacksonInvocationSerializer { private JacksonInvocationSerializer underTest; private static final String CLASS_NAME = "foo"; private static final String METHOD_NAME = "bar"; @BeforeEach void beforeEach() { ObjectMapper mapper = new ObjectMapper(); mapper.registerModule(new GuavaModule()); mapper.registerModule(new Jdk8Module()); mapper.registerModule(new JavaTimeModule()); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true); underTest = JacksonInvocationSerializer.builder() .mapper(mapper) .defaultInvocationSerializer( DefaultInvocationSerializer.builder() .serializableTypes(Set.of(Invocation.class)) .build()) .build(); } void check(Invocation invocation) { Invocation deserialized = serdeser(invocation); assertEquals(deserialized, serdeser(invocation)); assertEquals(invocation, deserialized); } Invocation serdeser(Invocation invocation) { try { var writer = new StringWriter(); underTest.serializeInvocation(invocation, writer); return underTest.deserializeInvocation(new StringReader(writer.toString())); } catch (IOException e) { throw new UncheckedIOException(e); } } @Test void testNoArgs() { check(new Invocation(String.class.getName(), "toString", new Class[0], new Object[0])); } @Test void testArrays() { check( new Invocation( CLASS_NAME, METHOD_NAME, new Class[] {int[].class}, new Object[] {new int[] {1, 2, 3}})); check( new Invocation( CLASS_NAME, METHOD_NAME, new Class[] {Integer[].class}, new Object[] {new Integer[] {1, 2, 3}})); check( new Invocation( CLASS_NAME, METHOD_NAME, new Class[] {String[].class}, new Object[] {new String[] {"1", "2", "3"}})); } @Test void testPrimitives() { Class[] primitives = { short.class, int.class, long.class, float.class, double.class, boolean.class, }; Object[] values = {(short) 2, 3, 4L, 1.23F, 1.23D, true}; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testBoxedPrimitives() { Class[] primitives = { Short.class, Integer.class, Long.class, Float.class, Double.class, Boolean.class, String.class }; Object[] values = { (Short) (short) 2, (Integer) 3, (Long) 4L, (Float) 1.23F, (Double) 1.23D, (Boolean) true, "Foo" }; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testJavaDateEnums() { Class[] primitives = {DayOfWeek.class, Month.class, ChronoUnit.class}; Object[] values = {DayOfWeek.FRIDAY, Month.APRIL, ChronoUnit.DAYS}; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testJavaUtilDate() { Class[] primitives = {Date.class}; Object[] values = {new Date()}; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void testJavaTimeClasses() { Class[] primitives = { Duration.class, Instant.class, LocalDate.class, LocalDateTime.class, MonthDay.class, Period.class, Year.class, YearMonth.class }; Object[] values = { Duration.ofDays(1), Instant.now().truncatedTo(ChronoUnit.MICROS), LocalDate.now(), LocalDateTime.now(), MonthDay.of(1, 1), Period.ofMonths(1), Year.now(), YearMonth.now() }; check(new Invocation(CLASS_NAME, METHOD_NAME, primitives, values)); } @Test void deserializes_new_representation_correctly() throws IOException { StringReader reader = new StringReader( "{\"c\":\"com.gruelbox.transactionoutbox.jackson.Service\",\"m\":\"parseDate\",\"p\":[\"String\"],\"a\":[{\"t\":\"String\",\"v\":\"2021-05-11\"}],\"x\":{\"REQUEST-ID\":\"someRequestId\"},\"s\":{\"SESSION-ID\":\"someSessionId\"}}"); Invocation invocation = underTest.deserializeInvocation(reader); assertEquals( new Invocation( "com.gruelbox.transactionoutbox.jackson.Service", "parseDate", new Class[] {String.class}, new Object[] {"2021-05-11"}, Map.of("REQUEST-ID", "someRequestId"), Map.of("SESSION-ID", "someSessionId")), invocation); } @Test void deserializes_old_representation_correctly() throws IOException { StringReader reader = new StringReader( "{\"c\":\"com.gruelbox.transactionoutbox.jackson.Service\",\"m\":\"parseDate\",\"p\":[\"String\"],\"a\":[{\"t\":\"String\",\"v\":\"2021-05-11\"}],\"x\":{\"REQUEST-ID\":\"someRequestId\"}}"); Invocation invocation = underTest.deserializeInvocation(reader); assertEquals( new Invocation( "com.gruelbox.transactionoutbox.jackson.Service", "parseDate", new Class[] {String.class}, new Object[] {"2021-05-11"}, Map.of("REQUEST-ID", "someRequestId"), null), invocation); } @Test void serializes_new_representation_stress_test() { Class[] parameterTypes = new Class[] {SerializationStressTestInput.class}; Object[] args = new Object[] {new SerializationStressTestInput()}; check(new Invocation(CLASS_NAME, METHOD_NAME, parameterTypes, args, null, null)); } @Test void serializes_new_representation_list() { Class[] parameterTypes = new Class[] {List.class}; Object[] args = new Object[] {List.of(MonetaryAmount.ofGbp("200"))}; check(new Invocation(CLASS_NAME, METHOD_NAME, parameterTypes, args, null, null)); } @Test void serializes_new_representation_set() { Class[] parameterTypes = new Class[] {Set.class}; Object[] args = new Object[] {Set.of(MonetaryAmount.ofGbp("200"))}; check(new Invocation(CLASS_NAME, METHOD_NAME, parameterTypes, args, null, null)); } @Test void serializes_new_representation_map() { Class[] parameterTypes = new Class[] {Set.class}; Object[] args = new Object[] {Map.of("investmentValue", MonetaryAmount.ofGbp("200"))}; check(new Invocation(CLASS_NAME, METHOD_NAME, parameterTypes, args, null, null)); } } ================================================ FILE: transactionoutbox-jackson/src/test/java/com/gruelbox/transactionoutbox/jackson/TestTransactionOutboxEntrySerialization.java ================================================ package com.gruelbox.transactionoutbox.jackson; import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.gruelbox.transactionoutbox.Invocation; import com.gruelbox.transactionoutbox.TransactionOutboxEntry; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; @Slf4j public class TestTransactionOutboxEntrySerialization { @Test void test() throws JsonProcessingException { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.setDefaultTyping(TransactionOutboxJacksonModule.typeResolver()); objectMapper.registerModule(new TransactionOutboxJacksonModule()); objectMapper.registerModule(new JavaTimeModule()); var entry = TransactionOutboxEntry.builder() .invocation( new Invocation( "c", "m", new Class[] {Map.class}, new Object[] { Map.of( "x", MonetaryAmount.ofGbp("200"), "y", 3, "z", List.of(1, 2, 3)) }, null, null)) .attempts(1) .blocked(true) .id("X") .description("Stuff") .nextAttemptTime(Instant.now().truncatedTo(ChronoUnit.MILLIS)) .uniqueRequestId("Y") .build(); var s = objectMapper.writeValueAsString(entry); var deserialized = objectMapper.readValue(s, TransactionOutboxEntry.class); assertEquals(entry, deserialized); } @Test void testWithSessionAndMdc() throws JsonProcessingException { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.setDefaultTyping(TransactionOutboxJacksonModule.typeResolver()); objectMapper.registerModule(new TransactionOutboxJacksonModule()); objectMapper.registerModule(new JavaTimeModule()); var entry = TransactionOutboxEntry.builder() .invocation( new Invocation( "c", "m", new Class[] {Map.class}, new Object[] { Map.of( "x", MonetaryAmount.ofGbp("200"), "y", 3, "z", List.of(1, 2, 3)) }, Map.of("a", "1"), Map.of("b", "2", "c", "3"))) .attempts(1) .blocked(true) .id("X") .description("Stuff") .nextAttemptTime(Instant.now().truncatedTo(ChronoUnit.MILLIS)) .uniqueRequestId("Y") .build(); var s = objectMapper.writeValueAsString(entry); log.info(s); var deserialized = objectMapper.readValue(s, TransactionOutboxEntry.class); assertEquals(entry, deserialized); } } ================================================ FILE: transactionoutbox-jackson/src/test/java/com/gruelbox/transactionoutbox/jackson/acceptance/TestJacksonSerializer.java ================================================ package com.gruelbox.transactionoutbox.jackson.acceptance; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.guava.GuavaModule; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.gruelbox.transactionoutbox.*; import com.gruelbox.transactionoutbox.jackson.JacksonInvocationSerializer; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import com.gruelbox.transactionoutbox.testing.LatchListener; import java.time.LocalDate; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; @Slf4j class TestJacksonSerializer extends AbstractAcceptanceTest { private final CountDownLatch latch = new CountDownLatch(1); private final ThreadLocal> sessionVariable = new ThreadLocal<>(); @Override protected Persistor persistor() { return DefaultPersistor.builder() .dialect(connectionDetails().dialect()) .serializer( JacksonInvocationSerializer.builder() .mapper( new ObjectMapper() .registerModule(new GuavaModule()) .registerModule(new Jdk8Module()) .registerModule(new JavaTimeModule()) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true)) .build()) .build(); } void process(List difficultDataStructure) { assertEquals(List.of(LocalDate.of(2000, 1, 1), "a", "b", 2), difficultDataStructure); assertEquals(Map.of("sessionVar", "foobar"), sessionVariable.get()); } @Test void testPolymorphicDeserialization() throws Exception { var transactionManager = txManager(); var outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(persistor()) .instantiator(Instantiator.using(clazz -> TestJacksonSerializer.this)) .listener( new LatchListener(latch) .andThen( new TransactionOutboxListener() { @Override public Map extractSession() { return Map.of("sessionVar", "foobar"); } @Override public void wrapInvocationAndInit(Invocator invocator) { sessionVariable.set(invocator.getInvocation().getSession()); try { invocator.runUnchecked(); } finally { sessionVariable.remove(); } } })) .build(); transactionManager.inTransaction( () -> outbox.schedule(getClass()).process(List.of(LocalDate.of(2000, 1, 1), "a", "b", 2))); assertTrue(latch.await(2, TimeUnit.SECONDS)); } } ================================================ FILE: transactionoutbox-jackson/src/test/resources/logback-test.xml ================================================ %d{HH:mm:ss.SSS} [%thread] %-5level SESSION-KEY=%X{SESSION-KEY} %logger{5} - %msg%n ================================================ FILE: transactionoutbox-jooq/README.md ================================================ # transaction-outbox-jooq [![jOOQ on Maven Central](https://maven-badges.herokuapp.com/maven-central/com.gruelbox/transactionoutbox-jooq/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.gruelbox/transactionoutbox-jooq) [![jOOQ Javadoc](https://www.javadoc.io/badge/com.gruelbox/transactionoutbox-jooq.svg?color=blue)](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-jooq) [![Latest snapshot](https://img.shields.io/github/v/tag/gruelbox/transaction-outbox?label=snapshot&sort=semver)](#development-snapshots) Extension for [transaction-outbox-core](../README.md) which integrates with jOOQ for transaction management. Like Transaction Outbox, jOOQ is intended to play nicely with any other transaction management approach, but provides its own as an option. If you are already using jOOQ's `TransactionProvider` via `DSLContext.transaction(...)` throughout your application, you can continue to do so with this extension. jOOQ gives you the option to either use thread-local transaction management or explicitly pass a contextual `DSLContext` or `Configuration` down your stack. You can do the same thing with `TransactionOutbox`. ## Installation ### Stable releases #### Maven ```xml com.gruelbox transactionoutbox-jooq 7.0.707 ``` #### Gradle ```groovy implementation 'com.gruelbox:transactionoutbox-jooq:7.0.707' ``` ### Development snapshots See [transactionoutbox-core](../README.md) for more information. ## Using thread-local transactions ### Configuration First, configure jOOQ to use thread-local transaction management: ```java var jooqConfig = new DefaultConfiguration(); var connectionProvider = new DataSourceConnectionProvider(dataSource); jooqConfig.setConnectionProvider(connectionProvider); jooqConfig.setSQLDialect(SQLDialect.H2); jooqConfig.setTransactionProvider(new ThreadLocalTransactionProvider(connectionProvider, true)); ``` Now connect `JooqTransactionListener`, which is the bridge between jOOQ and `TransactionOutbox`, and create the `DSLContext`: ```java var listener = JooqTransactionManager.createListener(); jooqConfig.set(listener); var dsl = DSL.using(jooqConfig); ``` Finally create the `TransactionOutbox`: ```java var outbox = TransactionOutbox.builder() .transactionManager(JooqTransactionManager.create(dsl, listener)) .persistor(Persistor.forDialect(Dialect.MY_SQL_8)) .build(); } ``` ### Usage You can now use jOOQ and Transaction Outbox together, assuming thread-bound transactions. ```java dsl.transaction(() -> { customerDao.save(new Customer(1L, "Martin", "Carthy")); customerDao.save(new Customer(2L, "Dave", "Pegg")); outbox.schedule(MyClass.class).publishCustomerCreatedEvent(1L); outbox.schedule(MyClass.class).publishCustomerCreatedEvent(2L); }); ``` ## Using explicit transaction context If you prefer not to use thread-local transactions, you are already taking on the burden of passing jOOQ `Configuration`s or `DSLContext`s down your stack. This is supported with `TransactionOutbox`, but requires a little explanation. ### Configuration Without the need to synchronise the thread context, setup is a bit easier: ```java // Create the DSLContext and connect the listener var dsl = DSL.using(dataSource, SQLDialect.H2); dsl.configuration().set(JooqTransactionManager.createListener()); // Create the outbox var outbox = TransactionOutbox.builder() .transactionManager(JooqTransactionManager.create(dsl)) .persistor(Persistor.forDialect(Dialect.MY_SQL_8)) .build(); ``` ### Usage The call pattern in the thread-local context example above will now not work: ```java outbox.schedule(MyClass.class).publishCustomerCreatedEvent(1L); ``` `TransactionOutbox` needs the currently active transaction context to write the database record. To do so, you need to change the scheduled method itself to receive a `Configuration`: ```java void publishCustomerCreatedEvent(long id, Configuration cfg2) { cfg.dsl().insertInto(...)... } ``` Then call accordingly: ```java dsl.transaction(cfg1 -> { new CustomerDao(cfg1).save(new Customer(1L, "Martin", "Carthy")); new CustomerDao(cfg1).save(new Customer(2L, "Dave", "Pegg")); outbox.schedule(MyClass.class).publishCustomerCreatedEvent(1L, cfg1); outbox.schedule(MyClass.class).publishCustomerCreatedEvent(2L, cfg1); }); ``` In the example above, `cfg1` is the transaction context in which the request is written to the database, and `cfg2` is the context in which it is executed, which will be a different transaction at some later time. `cfg1` is stripped from the request before writing it to the database and replaced with `cfg2` at run time. The reason for passing the `Configuration` in the scheduled method call itself (rather than the `schedule()` method call) is twofold: 1. It is very common for tasks to need access to the transaction context _at the time they are run_ in order to participate in that transaction. That way, if any part of the outbox task is rolled back, any work we do inside it is also rolled back. 2. If the method were not scheduled by `TransactionOutbox`, but instead called directly, it would need the `Configuration` passed to it anyway. By working this way we ensure that the API for calling directly or scheduled is the same, and therefore the two implementations are interchangeable. ================================================ FILE: transactionoutbox-jooq/pom.xml ================================================ transactionoutbox-parent com.gruelbox ${revision} 4.0.0 Transaction Outbox jOOQ jar transactionoutbox-jooq A safe implementation of the transactional outbox pattern for Java (jOOQ extension library) 5.2.4.RELEASE 21 21 com.gruelbox transactionoutbox-core ${project.version} org.jooq jooq 3.20.11 org.projectlombok lombok com.gruelbox transactionoutbox-testing ${project.version} test com.h2database h2 org.testcontainers testcontainers org.testcontainers junit-jupiter org.testcontainers postgresql org.testcontainers oracle-xe org.testcontainers mysql org.postgresql postgresql com.oracle.database.jdbc ojdbc11 com.mysql mysql-connector-j org.testcontainers mssqlserver com.microsoft.sqlserver mssql-jdbc ================================================ FILE: transactionoutbox-jooq/src/main/java/com/gruelbox/transactionoutbox/jooq/DefaultJooqTransactionManager.java ================================================ package com.gruelbox.transactionoutbox.jooq; import com.gruelbox.transactionoutbox.ParameterContextTransactionManager; import com.gruelbox.transactionoutbox.ThrowingTransactionalSupplier; import com.gruelbox.transactionoutbox.Transaction; import lombok.extern.slf4j.Slf4j; import org.jooq.Configuration; import org.jooq.DSLContext; /** * jOOQ transaction manager which uses explicitly-passed transaction context. Suitable for use with * {@link org.jooq.impl.DefaultTransactionProvider}. Relies on {@link JooqTransactionListener} being * connected to the {@link DSLContext}. */ @Slf4j final class DefaultJooqTransactionManager implements ParameterContextTransactionManager { private final DSLContext dsl; DefaultJooqTransactionManager(DSLContext dsl) { this.dsl = dsl; } @Override public T inTransactionReturnsThrows( ThrowingTransactionalSupplier work) { return dsl.transactionResult(cfg -> work.doWork(transactionFromContext(cfg))); } @Override public Transaction transactionFromContext(Configuration context) { Object txn = context.data(JooqTransactionListener.TXN_KEY); if (txn == null) { throw new IllegalStateException( JooqTransactionListener.class.getSimpleName() + " is not attached to the DSL"); } return (Transaction) txn; } @Override public Class contextType() { return Configuration.class; } } ================================================ FILE: transactionoutbox-jooq/src/main/java/com/gruelbox/transactionoutbox/jooq/JooqTransactionListener.java ================================================ package com.gruelbox.transactionoutbox.jooq; import com.gruelbox.transactionoutbox.spi.SimpleTransaction; import lombok.extern.slf4j.Slf4j; import org.jooq.TransactionContext; import org.jooq.TransactionListener; /** A jOOQ {@link TransactionListener} which synchronises a {@link JooqTransactionManager}. */ @Slf4j public class JooqTransactionListener implements TransactionListener { static final String TXN_KEY = JooqTransactionListener.class.getName() + ".txn"; private ThreadLocalJooqTransactionManager jooqTransactionManager; protected JooqTransactionListener() {} void setJooqTransactionManager(ThreadLocalJooqTransactionManager jooqTransactionManager) { this.jooqTransactionManager = jooqTransactionManager; } @Override public void beginStart(TransactionContext ctx) { // No-op } @Override public void beginEnd(TransactionContext ctx) { ctx.dsl() .connection( connection -> { SimpleTransaction transaction = new SimpleTransaction(connection, ctx.dsl().configuration()); ctx.dsl().configuration().data(TXN_KEY, transaction); if (jooqTransactionManager != null) { jooqTransactionManager.pushTransaction(transaction); } }); } @Override public void commitStart(TransactionContext ctx) { log.debug("Transaction commit start"); try { getTransaction(ctx).flushBatches(); } finally { getTransaction(ctx).close(); } } @Override public void commitEnd(TransactionContext ctx) { log.debug("Transaction commit end"); SimpleTransaction transaction = getTransaction(ctx); safePopThreadLocals(); transaction.processHooks(); } @Override public void rollbackStart(TransactionContext ctx) { log.debug("Transaction rollback"); getTransaction(ctx).close(); } @Override public void rollbackEnd(TransactionContext ctx) { safePopThreadLocals(); } private SimpleTransaction getTransaction(TransactionContext ctx) { return (SimpleTransaction) ctx.dsl().configuration().data(TXN_KEY); } private void safePopThreadLocals() { if (jooqTransactionManager != null) { jooqTransactionManager.popTransaction(); } } } ================================================ FILE: transactionoutbox-jooq/src/main/java/com/gruelbox/transactionoutbox/jooq/JooqTransactionManager.java ================================================ package com.gruelbox.transactionoutbox.jooq; import com.gruelbox.transactionoutbox.ParameterContextTransactionManager; import com.gruelbox.transactionoutbox.ThreadLocalContextTransactionManager; import com.gruelbox.transactionoutbox.TransactionManager; import com.gruelbox.transactionoutbox.TransactionOutbox; import org.jooq.Configuration; import org.jooq.DSLContext; /** * Transaction manager which uses jOOQ's transaction management. In order to wire into JOOQ's * transaction lifecycle, a slightly convoluted construction process is required which involves * first creating a {@link JooqTransactionListener}, including it in the JOOQ {@link Configuration} * while constructing the root {@link DSLContext}, and then finally linking the listener to the new * {@link JooqTransactionManager}: * * 
 * DataSourceConnectionProvider connectionProvider = new DataSourceConnectionProvider(dataSource);
 * DefaultConfiguration configuration = new DefaultConfiguration();
 * configuration.setConnectionProvider(connectionProvider);
 * configuration.setSQLDialect(SQLDialect.H2);
 * configuration.setTransactionProvider(new ThreadLocalTransactionProvider(connectionProvider));
 * JooqTransactionListener listener = JooqTransactionManager.createListener();
 * configuration.set(listener);
 * DSLContext dsl = DSL.using(configuration);
 * return JooqTransactionManager.create(dsl, listener);
*/ public interface JooqTransactionManager extends TransactionManager { /** * Creates the {@link org.jooq.TransactionListener} to wire into the {@link DSLContext}. See * class-level documentation for more detail. * * @return The transaction listener. */ static JooqTransactionListener createListener() { return new JooqTransactionListener(); } /** * Creates a transaction manager which uses thread-local context. Attaches to the supplied {@link * JooqTransactionListener} to receive notifications of transactions starting and finishing on the * local thread so that {@link TransactionOutbox#schedule(Class)} can be called for methods that * don't explicitly inject a {@link Configuration}, e.g.: * * 
dsl.transaction(() -> outbox.schedule(Foo.class).process("bar"));
* * @param dslContext The DSL context. * @param listener The listener, linked to the DSL context. * @return The transaction manager. */ static ThreadLocalContextTransactionManager create( DSLContext dslContext, JooqTransactionListener listener) { var result = new ThreadLocalJooqTransactionManager(dslContext); listener.setJooqTransactionManager(result); return result; } /** * Creates a transaction manager which uses explicitly-passed context, allowing multiple active * contexts in the current thread and contexts which are passed between threads. Requires a {@link * Configuration} for the transaction context or a {@link org.jooq.Transaction} to be used to be * passed any method called via {@link TransactionOutbox#schedule(Class)}. Example: * * 
   * void doSchedule() {
   *   // ctx1 is used to write the request to the DB
   *   dsl.transaction(ctx1 -> outbox.schedule(getClass()).process("bar", ctx1));
   * }
   *
   * // ctx2 is injected at run time
   * void process(String arg, org.jooq.Configuration ctx2) {
   *   ...
   * }
* *

Or: * * 

   * void doSchedule() {
   *   // tx1 is used to write the request to the DB
   *   transactionManager.inTransaction(tx1 -> outbox.schedule(getClass()).process("bar", tx1));
   * }
   *
   * // tx2 is injected at run time
   * void process(String arg, Transaction tx2) {
   *   ...
   * }
* * @param dslContext The DSL context. * @return The transaction manager. */ static ParameterContextTransactionManager create(DSLContext dslContext) { return new DefaultJooqTransactionManager(dslContext); } } ================================================ FILE: transactionoutbox-jooq/src/main/java/com/gruelbox/transactionoutbox/jooq/ThreadLocalJooqTransactionManager.java ================================================ package com.gruelbox.transactionoutbox.jooq; import com.gruelbox.transactionoutbox.ThrowingTransactionalSupplier; import com.gruelbox.transactionoutbox.spi.AbstractThreadLocalTransactionManager; import com.gruelbox.transactionoutbox.spi.SimpleTransaction; import lombok.extern.slf4j.Slf4j; import org.jooq.Configuration; import org.jooq.DSLContext; /** * jOOQ transaction manager which uses thread-local context. Best used with {@link * org.jooq.impl.ThreadLocalTransactionProvider}. Relies on a {@link JooqTransactionListener} being * attached to the {@link DSLContext}. */ @Slf4j final class ThreadLocalJooqTransactionManager extends AbstractThreadLocalTransactionManager implements JooqTransactionManager { private final DSLContext parentDsl; ThreadLocalJooqTransactionManager(DSLContext parentDsl) { this.parentDsl = parentDsl; } @Override public T inTransactionReturnsThrows( ThrowingTransactionalSupplier work) { DSLContext dsl = peekTransaction() .map(SimpleTransaction::context) .map(Configuration.class::cast) .map(Configuration::dsl) .orElse(parentDsl); return dsl.transactionResult( config -> config .dsl() .connectionResult(connection -> work.doWork(peekTransaction().orElseThrow()))); } } ================================================ FILE: transactionoutbox-jooq/src/test/java/com/gruelbox/transactionoutbox/jooq/acceptance/AbstractJooqAcceptanceTest.java ================================================ package com.gruelbox.transactionoutbox.jooq.acceptance; import com.gruelbox.transactionoutbox.TransactionManager; import com.gruelbox.transactionoutbox.testing.AbstractAcceptanceTest; import org.jooq.DSLContext; import org.junit.jupiter.api.Test; abstract class AbstractJooqAcceptanceTest extends AbstractAcceptanceTest { protected DSLContext dsl; @Override protected TransactionManager txManager() { throw new IllegalStateException("Needs to be defined"); } @Test abstract void testNestedDirectInvocation() throws Exception; @Test abstract void testNestedViaListener() throws Exception; @Test abstract void testNestedWithInnerFailure() throws Exception; } ================================================ FILE: transactionoutbox-jooq/src/test/java/com/gruelbox/transactionoutbox/jooq/acceptance/AbstractJooqAcceptanceThreadLocalTest.java ================================================ package com.gruelbox.transactionoutbox.jooq.acceptance; import static com.gruelbox.transactionoutbox.spi.Utils.uncheck; import static java.util.concurrent.CompletableFuture.runAsync; import static org.junit.jupiter.api.Assertions.*; import com.gruelbox.transactionoutbox.*; import com.gruelbox.transactionoutbox.jooq.JooqTransactionListener; import com.gruelbox.transactionoutbox.jooq.JooqTransactionManager; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.jooq.SQLDialect; import org.jooq.impl.DSL; import org.jooq.impl.DataSourceConnectionProvider; import org.jooq.impl.DefaultConfiguration; import org.jooq.impl.ThreadLocalTransactionProvider; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @Slf4j abstract class AbstractJooqAcceptanceThreadLocalTest extends AbstractJooqAcceptanceTest { private ThreadLocalContextTransactionManager txm; @Override protected final ThreadLocalContextTransactionManager txManager() { return txm; } protected SQLDialect jooqDialect() { return SQLDialect.H2; } @BeforeEach final void beforeEach() { DataSourceConnectionProvider connectionProvider = new DataSourceConnectionProvider(dataSource); DefaultConfiguration configuration = new DefaultConfiguration(); configuration.setConnectionProvider(connectionProvider); configuration.setSQLDialect(jooqDialect()); configuration.setTransactionProvider( new ThreadLocalTransactionProvider(connectionProvider, true)); JooqTransactionListener listener = JooqTransactionManager.createListener(); configuration.set(listener); dsl = DSL.using(configuration); txm = JooqTransactionManager.create(dsl, listener); JooqTestUtils.createTestTable(dsl); } @Test @Override void testNestedDirectInvocation() throws Exception { CountDownLatch latch1 = new CountDownLatch(1); CountDownLatch latch2 = new CountDownLatch(1); ThreadLocalContextTransactionManager transactionManager = txManager(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(connectionDetails().dialect())) .instantiator(Instantiator.using(clazz -> new Worker(transactionManager))) .attemptFrequency(Duration.of(1, ChronoUnit.SECONDS)) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { if (entry.getInvocation().getArgs()[0].equals(1)) { latch1.countDown(); } else { latch2.countDown(); } } }) .build(); clearOutbox(); withRunningFlusher( outbox, () -> { transactionManager.inTransactionThrows( tx1 -> { outbox.schedule(Worker.class).process(1); transactionManager.inTransactionThrows( tx2 -> outbox.schedule(Worker.class).process(2)); // Neither should be fired - the second job is in a nested transaction CompletableFuture.allOf( runAsync( () -> uncheck(() -> assertFalse(latch1.await(2, TimeUnit.SECONDS)))), runAsync( () -> uncheck(() -> assertFalse(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); // Should be fired after commit CompletableFuture.allOf( runAsync(() -> uncheck(() -> assertTrue(latch1.await(2, TimeUnit.SECONDS)))), runAsync(() -> uncheck(() -> assertTrue(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); JooqTestUtils.assertRecordExists(dsl, 1); JooqTestUtils.assertRecordExists(dsl, 2); } @Test @Override void testNestedViaListener() throws Exception { CountDownLatch latch1 = new CountDownLatch(1); CountDownLatch latch2 = new CountDownLatch(1); ThreadLocalContextTransactionManager transactionManager = txManager(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(connectionDetails().dialect())) .instantiator(Instantiator.using(clazz -> new Worker(transactionManager))) .attemptFrequency(Duration.of(1, ChronoUnit.SECONDS)) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { if (entry.getInvocation().getArgs()[0].equals(1)) { latch1.countDown(); } else { latch2.countDown(); } } }) .build(); clearOutbox(); withRunningFlusher( outbox, () -> { dsl.transaction( ctx -> { outbox.schedule(Worker.class).process(1); ctx.dsl().transaction(() -> outbox.schedule(Worker.class).process(2)); // Neither should be fired - the second job is in a nested transaction CompletableFuture.allOf( runAsync( () -> uncheck(() -> assertFalse(latch1.await(2, TimeUnit.SECONDS)))), runAsync( () -> uncheck(() -> assertFalse(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); // Both should be fired after commit CompletableFuture.allOf( runAsync(() -> uncheck(() -> assertTrue(latch1.await(10, TimeUnit.SECONDS)))), runAsync(() -> uncheck(() -> assertTrue(latch2.await(10, TimeUnit.SECONDS))))) .get(); }); JooqTestUtils.assertRecordExists(dsl, 1); JooqTestUtils.assertRecordExists(dsl, 2); } /** * Ensures that given the rollback of an inner transaction, any outbox work scheduled in the inner * transaction is rolled back while the outer transaction's works. */ @Test @Override void testNestedWithInnerFailure() throws Exception { CountDownLatch latch1 = new CountDownLatch(1); CountDownLatch latch2 = new CountDownLatch(1); ThreadLocalContextTransactionManager transactionManager = txManager(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(connectionDetails().dialect())) .instantiator(Instantiator.using(clazz -> new Worker(transactionManager))) .attemptFrequency(Duration.of(1, ChronoUnit.SECONDS)) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { if (entry.getInvocation().getArgs()[0].equals(1)) { latch1.countDown(); } else { latch2.countDown(); } } }) .build(); clearOutbox(); withRunningFlusher( outbox, () -> { dsl.transaction( ctx -> { outbox.schedule(Worker.class).process(1); assertThrows( UnsupportedOperationException.class, () -> ctx.dsl() .transaction( () -> { outbox.schedule(Worker.class).process(2); throw new UnsupportedOperationException(); })); CompletableFuture.allOf( runAsync( () -> uncheck(() -> assertFalse(latch1.await(2, TimeUnit.SECONDS)))), runAsync( () -> uncheck(() -> assertFalse(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); CompletableFuture.allOf( runAsync(() -> uncheck(() -> assertTrue(latch1.await(2, TimeUnit.SECONDS)))), runAsync(() -> uncheck(() -> assertFalse(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); } @SuppressWarnings("EmptyMethod") static class Worker { private final ThreadLocalContextTransactionManager transactionManager; Worker(ThreadLocalContextTransactionManager transactionManager) { this.transactionManager = transactionManager; } @SuppressWarnings("SameParameterValue") void process(int i) { JooqTestUtils.writeRecord(transactionManager, i); } } } ================================================ FILE: transactionoutbox-jooq/src/test/java/com/gruelbox/transactionoutbox/jooq/acceptance/JooqTestUtils.java ================================================ package com.gruelbox.transactionoutbox.jooq.acceptance; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.gruelbox.transactionoutbox.ThreadLocalContextTransactionManager; import com.gruelbox.transactionoutbox.Transaction; import lombok.extern.slf4j.Slf4j; import org.jooq.Configuration; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Table; import org.jooq.impl.DSL; import org.jooq.impl.SQLDataType; @Slf4j class JooqTestUtils { private static final Table TEST_TABLE = DSL.table("TEST_TABLE_JOOQ"); private static final String VAL = "val"; static void createTestTable(DSLContext dsl) { log.info("Creating table"); dsl.dropTableIfExists(TEST_TABLE).execute(); dsl.createTable(TEST_TABLE).column(VAL, SQLDataType.INTEGER).execute(); } static void writeRecord(Configuration configuration, int value) { log.info("Inserting record {}", value); configuration.dsl().insertInto(TEST_TABLE).values(value).execute(); } static void writeRecord(Transaction transaction, int value) { Configuration configuration = transaction.context(); writeRecord(configuration, value); } static void writeRecord(ThreadLocalContextTransactionManager transactionManager, int value) { transactionManager.requireTransaction(tx -> writeRecord(tx, value)); } static void assertRecordExists(DSLContext dsl, int value) { assertTrue( dsl.select().from(TEST_TABLE).where(DSL.field(VAL).eq(value)).fetchOptional().isPresent()); } static void assertRecordNotExists( DSLContext dsl, @SuppressWarnings("SameParameterValue") int value) { assertFalse( dsl.select().from(TEST_TABLE).where(DSL.field(VAL).eq(value)).fetchOptional().isPresent()); } } ================================================ FILE: transactionoutbox-jooq/src/test/java/com/gruelbox/transactionoutbox/jooq/acceptance/TestJooqThreadLocalH2.java ================================================ package com.gruelbox.transactionoutbox.jooq.acceptance; import lombok.extern.slf4j.Slf4j; @Slf4j class TestJooqThreadLocalH2 extends AbstractJooqAcceptanceThreadLocalTest {} ================================================ FILE: transactionoutbox-jooq/src/test/java/com/gruelbox/transactionoutbox/jooq/acceptance/TestJooqThreadLocalMSSqlServer2019.java ================================================ package com.gruelbox.transactionoutbox.jooq.acceptance; import com.gruelbox.transactionoutbox.Dialect; import java.time.Duration; import lombok.extern.slf4j.Slf4j; import org.jooq.SQLDialect; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @Slf4j @Testcontainers class TestJooqThreadLocalMSSqlServer2019 extends AbstractJooqAcceptanceThreadLocalTest { @Container @SuppressWarnings({"rawtypes", "resource", "unchecked"}) private static final JdbcDatabaseContainer container = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest") .acceptLicense() .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.MS_SQL_SERVER) .driverClassName("com.microsoft.sqlserver.jdbc.SQLServerDriver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } @Override protected SQLDialect jooqDialect() { return SQLDialect.DEFAULT; } } ================================================ FILE: transactionoutbox-jooq/src/test/java/com/gruelbox/transactionoutbox/jooq/acceptance/TestJooqThreadLocalMySql5.java ================================================ package com.gruelbox.transactionoutbox.jooq.acceptance; import com.gruelbox.transactionoutbox.Dialect; import java.time.Duration; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.jooq.SQLDialect; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @Slf4j @Testcontainers class TestJooqThreadLocalMySql5 extends AbstractJooqAcceptanceThreadLocalTest { @Container @SuppressWarnings({"rawtypes", "resource", "unchecked"}) private static final JdbcDatabaseContainer container = (JdbcDatabaseContainer) new MySQLContainer("mysql:5") .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true) .withTmpFs(Map.of("/var/lib/mysql", "rw")); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.MY_SQL_5) .driverClassName("com.mysql.cj.jdbc.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } @Override protected SQLDialect jooqDialect() { return SQLDialect.MYSQL; } } ================================================ FILE: transactionoutbox-jooq/src/test/java/com/gruelbox/transactionoutbox/jooq/acceptance/TestJooqThreadLocalMySql8.java ================================================ package com.gruelbox.transactionoutbox.jooq.acceptance; import com.gruelbox.transactionoutbox.*; import java.time.Duration; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.jooq.SQLDialect; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @Slf4j @Testcontainers class TestJooqThreadLocalMySql8 extends AbstractJooqAcceptanceThreadLocalTest { @Container @SuppressWarnings({"rawtypes", "resource", "unchecked"}) private static final JdbcDatabaseContainer container = (JdbcDatabaseContainer) new MySQLContainer("mysql:8") .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true) .withTmpFs(Map.of("/var/lib/mysql", "rw")); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.MY_SQL_8) .driverClassName("com.mysql.cj.jdbc.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } @Override protected SQLDialect jooqDialect() { return SQLDialect.MYSQL; } } ================================================ FILE: transactionoutbox-jooq/src/test/java/com/gruelbox/transactionoutbox/jooq/acceptance/TestJooqThreadLocalPostgres16.java ================================================ package com.gruelbox.transactionoutbox.jooq.acceptance; import com.gruelbox.transactionoutbox.Dialect; import java.time.Duration; import lombok.extern.slf4j.Slf4j; import org.jooq.SQLDialect; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @Slf4j @Testcontainers class TestJooqThreadLocalPostgres16 extends AbstractJooqAcceptanceThreadLocalTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = (JdbcDatabaseContainer) new PostgreSQLContainer("postgres:16") .withStartupTimeout(Duration.ofHours(1)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.POSTGRESQL_9) .driverClassName("org.postgresql.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } @Override protected SQLDialect jooqDialect() { return SQLDialect.POSTGRES; } } ================================================ FILE: transactionoutbox-jooq/src/test/java/com/gruelbox/transactionoutbox/jooq/acceptance/TestJooqTransactionManagerWithDefaultProviderAndExplicitlyPassedContext.java ================================================ package com.gruelbox.transactionoutbox.jooq.acceptance; import static com.gruelbox.transactionoutbox.jooq.acceptance.JooqTestUtils.createTestTable; import static com.gruelbox.transactionoutbox.spi.Utils.uncheck; import static com.gruelbox.transactionoutbox.testing.TestUtils.runSql; import static java.util.concurrent.CompletableFuture.runAsync; import static org.junit.jupiter.api.Assertions.*; import com.gruelbox.transactionoutbox.*; import com.gruelbox.transactionoutbox.jooq.JooqTransactionManager; import com.gruelbox.transactionoutbox.testing.LatchListener; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.Map; import java.util.UUID; import java.util.concurrent.*; import lombok.extern.slf4j.Slf4j; import org.jooq.Configuration; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.jooq.impl.DSL; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @Slf4j class TestJooqTransactionManagerWithDefaultProviderAndExplicitlyPassedContext { private HikariDataSource dataSource; private DSLContext dsl; private static ThreadLocal sessionVar = new ThreadLocal<>(); @BeforeEach void beforeEach() { TestingMode.enable(); dataSource = pooledDataSource(); dsl = createDsl(); createTestTable(dsl); } @AfterEach void afterEach() { TestingMode.disable(); dataSource.close(); } private HikariDataSource pooledDataSource() { HikariConfig config = new HikariConfig(); config.setJdbcUrl( "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DEFAULT_LOCK_TIMEOUT=2000;LOB_TIMEOUT=2000;MV_STORE=TRUE;DATABASE_TO_UPPER=FALSE"); config.setUsername("test"); config.setPassword("test"); config.addDataSourceProperty("cachePrepStmts", "true"); return new HikariDataSource(config); } private DSLContext createDsl() { dsl = DSL.using(dataSource, SQLDialect.H2); dsl.configuration().set(JooqTransactionManager.createListener()); return dsl; } private TransactionManager createTransactionManager() { return JooqTransactionManager.create(dsl); } @Test void testSimplePassingTransaction() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(createTransactionManager()) .persistor(Persistor.forDialect(Dialect.H2)) .listener(new LatchListener(latch)) .build(); clearOutbox(createTransactionManager()); createTransactionManager() .inTransaction( tx -> { outbox.schedule(Worker.class).process(1, tx); try { // Should not be fired until after commit assertFalse(latch.await(2, TimeUnit.SECONDS)); } catch (InterruptedException e) { fail("Interrupted"); } }); // Should be fired after commit assertTrue(latch.await(2, TimeUnit.SECONDS)); JooqTestUtils.assertRecordExists(dsl, 1); } @Test void testSimplePassingContext() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(createTransactionManager()) .persistor(Persistor.forDialect(Dialect.H2)) .listener(new LatchListener(latch)) .build(); clearOutbox(createTransactionManager()); DSLContext dsl = createDsl(); dsl.transaction( cx1 -> { outbox.schedule(Worker.class).process(1, cx1); try { // Should not be fired until after commit assertFalse(latch.await(2, TimeUnit.SECONDS)); } catch (InterruptedException e) { fail("Interrupted"); } }); // Should be fired after commit assertTrue(latch.await(2, TimeUnit.SECONDS)); JooqTestUtils.assertRecordExists(dsl, 1); } @Test void testNestedPassingContext() throws Exception { CountDownLatch latch1 = new CountDownLatch(1); CountDownLatch latch2 = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(createTransactionManager()) .persistor(Persistor.forDialect(Dialect.H2)) .attemptFrequency(Duration.of(1, ChronoUnit.SECONDS)) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { if (entry.getInvocation().getArgs()[0].equals(1)) { latch1.countDown(); } else { latch2.countDown(); } } }) .build(); clearOutbox(createTransactionManager()); withRunningFlusher( outbox, () -> { dsl.transaction( ctx -> { outbox.schedule(Worker.class).process(1, ctx); ctx.dsl().transaction(cx1 -> outbox.schedule(Worker.class).process(2, ctx)); // Neither should be fired - the second job is in a nested transaction CompletableFuture.allOf( runAsync( () -> uncheck(() -> assertFalse(latch1.await(2, TimeUnit.SECONDS)))), runAsync( () -> uncheck(() -> assertFalse(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); // Both should be fired after commit CompletableFuture.allOf( runAsync(() -> uncheck(() -> assertTrue(latch1.await(2, TimeUnit.SECONDS)))), runAsync(() -> uncheck(() -> assertTrue(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); JooqTestUtils.assertRecordExists(dsl, 1); JooqTestUtils.assertRecordExists(dsl, 2); } @Test void testNestedPassingTransaction() throws Exception { CountDownLatch latch1 = new CountDownLatch(1); CountDownLatch latch2 = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(createTransactionManager()) .persistor(Persistor.forDialect(Dialect.H2)) .attemptFrequency(Duration.of(1, ChronoUnit.SECONDS)) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { if (entry.getInvocation().getArgs()[0].equals(1)) { latch1.countDown(); } else { latch2.countDown(); } } }) .build(); clearOutbox(createTransactionManager()); withRunningFlusher( outbox, () -> { createTransactionManager() .inTransactionThrows( tx1 -> { outbox.schedule(Worker.class).process(1, tx1); createTransactionManager() .inTransactionThrows(tx2 -> outbox.schedule(Worker.class).process(2, tx2)); // The inner transaction should be committed - these are different semantics CompletableFuture.allOf( runAsync( () -> uncheck(() -> assertFalse(latch1.await(2, TimeUnit.SECONDS)))), runAsync( () -> uncheck(() -> assertTrue(latch2.await(2, TimeUnit.SECONDS))))) .get(); JooqTestUtils.assertRecordExists(dsl, 2); }); // Should be fired after commit assertTrue(latch1.await(2, TimeUnit.SECONDS)); }); JooqTestUtils.assertRecordExists(dsl, 1); } /** * Ensures that given the rollback of an inner transaction, any outbox work scheduled in the inner * transaction is rolled back while the outer transaction's works. */ @Test void testNestedWithInnerFailure() throws Exception { CountDownLatch latch1 = new CountDownLatch(1); CountDownLatch latch2 = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(createTransactionManager()) .persistor(Persistor.forDialect(Dialect.H2)) .attemptFrequency(Duration.of(1, ChronoUnit.SECONDS)) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { if (entry.getInvocation().getArgs()[0].equals(1)) { latch1.countDown(); } else { latch2.countDown(); } } }) .build(); clearOutbox(createTransactionManager()); withRunningFlusher( outbox, () -> { dsl.transaction( ctx -> { outbox.schedule(Worker.class).process(1, ctx); assertThrows( UnsupportedOperationException.class, () -> ctx.dsl() .transaction( cx2 -> { outbox.schedule(Worker.class).process(2, cx2); throw new UnsupportedOperationException(); })); CompletableFuture.allOf( runAsync( () -> uncheck(() -> assertFalse(latch1.await(2, TimeUnit.SECONDS)))), runAsync( () -> uncheck(() -> assertFalse(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); CompletableFuture.allOf( runAsync(() -> uncheck(() -> assertTrue(latch1.await(2, TimeUnit.SECONDS)))), runAsync(() -> uncheck(() -> assertFalse(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); JooqTestUtils.assertRecordExists(dsl, 1); JooqTestUtils.assertRecordNotExists(dsl, 2); } @Test void testSessionVariables() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); var sessionVarLocal = UUID.randomUUID().toString(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(createTransactionManager()) .persistor(Persistor.forDialect(Dialect.H2)) .listener( new LatchListener(latch) .andThen( new TransactionOutboxListener() { @Override public Map extractSession() { return Map.of("sesvar", sessionVarLocal); } @Override public void wrapInvocationAndInit(Invocator invocator) { sessionVar.set(invocator.getInvocation().getSession().get("sesvar")); try { invocator.runUnchecked(); } finally { sessionVar.remove(); } } })) .build(); clearOutbox(createTransactionManager()); createTransactionManager() .inTransaction( tx -> outbox.schedule(Worker.class).checkSessionPresent(sessionVarLocal, tx)); // Should be fired after commit assertTrue(latch.await(2, TimeUnit.SECONDS)); } private void clearOutbox(TransactionManager transactionManager) { runSql(transactionManager, "DELETE FROM TXNO_OUTBOX"); } private void withRunningFlusher(TransactionOutbox outbox, ThrowingRunnable runnable) throws Exception { ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); try { scheduler.scheduleAtFixedRate( () -> { if (Thread.interrupted()) { return; } outbox.flush(); }, 500, 500, TimeUnit.MILLISECONDS); runnable.run(); } finally { scheduler.shutdown(); assertTrue(scheduler.awaitTermination(20, TimeUnit.SECONDS)); } } @SuppressWarnings("EmptyMethod") static class Worker { @SuppressWarnings("SameParameterValue") void process(int i, Transaction transaction) { JooqTestUtils.writeRecord(transaction, i); } void checkSessionPresent(String expected, Transaction transaction) { assertEquals(expected, sessionVar.get()); } void process(int i, Configuration configuration) { JooqTestUtils.writeRecord(configuration, i); } } } ================================================ FILE: transactionoutbox-jooq/src/test/java/com/gruelbox/transactionoutbox/jooq/acceptance/TestJooqTransactionManagerWithDefaultProviderAndThreadLocalContext.java ================================================ package com.gruelbox.transactionoutbox.jooq.acceptance; import static com.gruelbox.transactionoutbox.spi.Utils.uncheck; import static com.gruelbox.transactionoutbox.testing.TestUtils.runSql; import static java.util.concurrent.CompletableFuture.runAsync; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; import static org.junit.jupiter.api.Assertions.*; import com.gruelbox.transactionoutbox.*; import com.gruelbox.transactionoutbox.jooq.JooqTransactionListener; import com.gruelbox.transactionoutbox.jooq.JooqTransactionManager; import com.gruelbox.transactionoutbox.testing.LatchListener; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.Map; import java.util.UUID; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; import lombok.extern.slf4j.Slf4j; import org.hamcrest.MatcherAssert; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.jooq.impl.DSL; import org.jooq.impl.DataSourceConnectionProvider; import org.jooq.impl.DefaultConfiguration; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @Slf4j class TestJooqTransactionManagerWithDefaultProviderAndThreadLocalContext { private static ThreadLocal sessionVar = new ThreadLocal<>(); private final ExecutorService unreliablePool = new ThreadPoolExecutor(2, 2, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(16)); private HikariDataSource dataSource; private ThreadLocalContextTransactionManager transactionManager; private DSLContext dsl; @BeforeEach void beforeEach() { TestingMode.enable(); dataSource = pooledDataSource(); transactionManager = createTransactionManager(); JooqTestUtils.createTestTable(dsl); } @AfterEach void afterEach() { TestingMode.disable(); dataSource.close(); } private HikariDataSource pooledDataSource() { HikariConfig config = new HikariConfig(); config.setJdbcUrl( "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DEFAULT_LOCK_TIMEOUT=2000;LOB_TIMEOUT=2000;MV_STORE=TRUE;DATABASE_TO_UPPER=FALSE"); config.setUsername("test"); config.setPassword("test"); config.addDataSourceProperty("cachePrepStmts", "true"); return new HikariDataSource(config); } private ThreadLocalContextTransactionManager createTransactionManager() { DefaultConfiguration configuration = new DefaultConfiguration(); configuration.setConnectionProvider(new DataSourceConnectionProvider(dataSource)); configuration.setSQLDialect(SQLDialect.H2); JooqTransactionListener listener = JooqTransactionManager.createListener(); configuration.set(listener); dsl = DSL.using(configuration); return JooqTransactionManager.create(dsl, listener); } @Test void testSimpleDirectInvocation() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(Dialect.H2)) .instantiator(Instantiator.using(clazz -> new Worker(transactionManager))) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { latch.countDown(); } }) .build(); clearOutbox(transactionManager); transactionManager.inTransaction( () -> { outbox.schedule(Worker.class).process(1); try { // Should not be fired until after commit assertFalse(latch.await(2, TimeUnit.SECONDS)); } catch (InterruptedException e) { fail("Interrupted"); } }); // Should be fired after commit assertTrue(latch.await(2, TimeUnit.SECONDS)); JooqTestUtils.assertRecordExists(dsl, 1); } @Test void testSimpleViaListener() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .instantiator(Instantiator.using(clazz -> new Worker(transactionManager))) .persistor(Persistor.forDialect(Dialect.H2)) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { latch.countDown(); } }) .build(); clearOutbox(transactionManager); dsl.transaction( cx1 -> { outbox.schedule(Worker.class).process(1); try { // Should not be fired until after commit assertFalse(latch.await(2, TimeUnit.SECONDS)); } catch (InterruptedException e) { fail("Interrupted"); } }); // Should be fired after commit assertTrue(latch.await(2, TimeUnit.SECONDS)); JooqTestUtils.assertRecordExists(dsl, 1); } @Test void testNestedViaListener() throws Exception { CountDownLatch latch1 = new CountDownLatch(1); CountDownLatch latch2 = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(Dialect.H2)) .instantiator(Instantiator.using(clazz -> new Worker(transactionManager))) .attemptFrequency(Duration.of(1, ChronoUnit.SECONDS)) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { if (entry.getInvocation().getArgs()[0].equals(1)) { latch1.countDown(); } else { latch2.countDown(); } } }) .build(); clearOutbox(transactionManager); withRunningFlusher( outbox, () -> { dsl.transaction( ctx -> { outbox.schedule(Worker.class).process(1); ctx.dsl().transaction(cx1 -> outbox.schedule(Worker.class).process(2)); // Neither should be fired - the second job is in a nested transaction CompletableFuture.allOf( runAsync( () -> uncheck(() -> assertFalse(latch1.await(2, TimeUnit.SECONDS)))), runAsync( () -> uncheck(() -> assertFalse(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); // Both should be fired after commit CompletableFuture.allOf( runAsync(() -> uncheck(() -> assertTrue(latch1.await(2, TimeUnit.SECONDS)))), runAsync(() -> uncheck(() -> assertTrue(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); JooqTestUtils.assertRecordExists(dsl, 1); JooqTestUtils.assertRecordExists(dsl, 2); } /** * Ensures that given the rollback of an inner transaction, any outbox work scheduled in the inner * transaction is rolled back while the outer transaction's works. */ @Test void testNestedWithInnerFailure() throws Exception { CountDownLatch latch1 = new CountDownLatch(1); CountDownLatch latch2 = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(Dialect.H2)) .instantiator(Instantiator.using(clazz -> new Worker(transactionManager))) .attemptFrequency(Duration.of(1, ChronoUnit.SECONDS)) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { if (entry.getInvocation().getArgs()[0].equals(1)) { latch1.countDown(); } else { latch2.countDown(); } } }) .build(); clearOutbox(transactionManager); withRunningFlusher( outbox, () -> { dsl.transaction( ctx -> { outbox.schedule(Worker.class).process(1); assertThrows( UnsupportedOperationException.class, () -> ctx.dsl() .transaction( cx2 -> { outbox.schedule(Worker.class).process(2); throw new UnsupportedOperationException(); })); CompletableFuture.allOf( runAsync( () -> uncheck(() -> assertFalse(latch1.await(2, TimeUnit.SECONDS)))), runAsync( () -> uncheck(() -> assertFalse(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); CompletableFuture.allOf( runAsync(() -> uncheck(() -> assertTrue(latch1.await(2, TimeUnit.SECONDS)))), runAsync(() -> uncheck(() -> assertFalse(latch2.await(2, TimeUnit.SECONDS))))) .get(); }); } @Test void retryBehaviour() throws Exception { CountDownLatch latch = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(Dialect.H2)) .instantiator(new FailingInstantiator()) .submitter(Submitter.withExecutor(unreliablePool)) .attemptFrequency(Duration.ofSeconds(1)) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { latch.countDown(); } }) .build(); clearOutbox(transactionManager); withRunningFlusher( outbox, () -> { transactionManager.inTransaction(() -> outbox.schedule(InterfaceWorker.class).process(3)); assertTrue(latch.await(15, TimeUnit.SECONDS)); }); } @Test void highVolumeUnreliable() throws Exception { int count = 10; CountDownLatch latch = new CountDownLatch(count * 10); ConcurrentHashMap results = new ConcurrentHashMap<>(); ConcurrentHashMap duplicates = new ConcurrentHashMap<>(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(Dialect.H2)) .instantiator(new FailingInstantiator()) .submitter(Submitter.withExecutor(unreliablePool)) .attemptFrequency(Duration.ofSeconds(1)) .flushBatchSize(1000) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { Integer i = (Integer) entry.getInvocation().getArgs()[0]; if (results.putIfAbsent(i, i) != null) { duplicates.put(i, i); } latch.countDown(); } }) .build(); withRunningFlusher( outbox, () -> { IntStream.range(0, count) .parallel() .forEach( i -> dsl.transaction( cx1 -> { for (int j = 0; j < 10; j++) { outbox.schedule(InterfaceWorker.class).process(i * 10 + j); } })); assertTrue(latch.await(30, TimeUnit.SECONDS)); }); MatcherAssert.assertThat( "Should never get duplicates running to full completion", duplicates.keySet(), empty()); MatcherAssert.assertThat( "Only got: " + results.keySet(), results.keySet(), containsInAnyOrder(IntStream.range(0, count * 10).boxed().toArray())); } @Test void testSessionVariables() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); var sessionVarLocal = UUID.randomUUID().toString(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .instantiator(Instantiator.using(clazz -> new Worker(transactionManager))) .persistor(Persistor.forDialect(Dialect.H2)) .listener( new LatchListener(latch) .andThen( new TransactionOutboxListener() { @Override public Map extractSession() { return Map.of("sesvar", sessionVarLocal); } @Override public void wrapInvocationAndInit(Invocator invocator) { sessionVar.set(invocator.getInvocation().getSession().get("sesvar")); try { invocator.runUnchecked(); } finally { sessionVar.remove(); } } })) .build(); clearOutbox(transactionManager); transactionManager.inTransaction( tx -> outbox.schedule(Worker.class).checkSessionPresent(sessionVarLocal)); // Should be fired after commit assertTrue(latch.await(2, TimeUnit.SECONDS)); } private void clearOutbox(TransactionManager transactionManager) { runSql(transactionManager, "DELETE FROM TXNO_OUTBOX"); } private void withRunningFlusher(TransactionOutbox outbox, ThrowingRunnable runnable) throws Exception { ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); try { scheduler.scheduleAtFixedRate( () -> { if (Thread.interrupted()) { return; } outbox.flush(); }, 500, 500, TimeUnit.MILLISECONDS); runnable.run(); } finally { scheduler.shutdown(); assertTrue(scheduler.awaitTermination(20, TimeUnit.SECONDS)); } } interface InterfaceWorker { void process(int i); } @SuppressWarnings("EmptyMethod") static class Worker { private final ThreadLocalContextTransactionManager transactionManager; Worker(ThreadLocalContextTransactionManager transactionManager) { this.transactionManager = transactionManager; } @SuppressWarnings("SameParameterValue") void process(int i) { JooqTestUtils.writeRecord(transactionManager, i); } void checkSessionPresent(String expected) { assertEquals(expected, sessionVar.get()); } } private static class FailingInstantiator implements Instantiator { private final AtomicInteger attempts; FailingInstantiator() { this.attempts = new AtomicInteger(0); } @Override public String getName(Class clazz) { return clazz.getName(); } @Override public Object getInstance(String name) { return (InterfaceWorker) (i) -> { if (attempts.incrementAndGet() < 3) { throw new RuntimeException("Temporary failure"); } }; } } } ================================================ FILE: transactionoutbox-jooq/src/test/resources/logback-test.xml ================================================ %d{HH:mm:ss.SSS} [%thread] %-5level SESSION-KEY=%X{SESSION-KEY} %logger{5} - %msg%n ================================================ FILE: transactionoutbox-quarkus/README.md ================================================ # transactionoutbox-quarkus Extension for [transaction-outbox-core](../README.md) which integrates CDI's DI and Quarkus transaction management. Tested with Quarkus implementation (Arc/Agroal) ## Installation ### Stable releases The latest stable release is available from Maven Central. #### Maven ```xml com.gruelbox transactionoutbox-quarkus 7.0.707 ``` #### Gradle ```groovy implementation 'com.gruelbox:transactionoutbox-quarkus:7.0.707' ``` ### Development snapshots Maven Central is updated regularly. Alternatively, if you want to stay at the bleeding edge, you can use continuously-delivered releases from [Github Package Repository](https://github.com/gruelbox/transaction-outbox/packages). These can be used from production builds since they will never be deleted. #### Maven ```xml github-transaction-outbox Gruelbox Github Repository https://maven.pkg.github.com/gruelbox/transaction-outbox ``` You will need to authenticate with Github to use Github Package Repository. Create a personal access token in [your GitHub settings](https://github.com/settings/tokens). It only needs **read:package** permissions. Then add something like the following in your Maven `settings.xml`: ```xml github-transaction-outbox ${env.GITHUB_USERNAME} ${env.GITHUB_TOKEN} ``` The above example uses environment variables, allowing you to keep the credentials out of source control, but you can hard-code them if you know what you're doing. ## Configuration Create your `TransactionOutbox` as a bean: ```java @Produces public TransactionOutbox transactionOutbox(QuarkusTransactionManager transactionManager) { return TransactionOutbox.builder().instantiator(CdiInstantiator.create()).transactionManager(transactionManager).persistor(Persistor.forDialect(Dialect.H2)).build(); } ``` ## Usage ```java @Transactional public void doStuff() { customerRepository.save(new Customer(1L, "Martin", "Carthy")); customerRepository.save(new Customer(2L, "Dave", "Pegg")); outbox.get().schedule(getClass()).publishCustomerCreatedEvent(1L); outbox.get().schedule(getClass()).publishCustomerCreatedEvent(2L); } void publishCustomerCreatedEvent(long id) { // Remote call here } ``` ================================================ FILE: transactionoutbox-quarkus/pom.xml ================================================ transactionoutbox-parent com.gruelbox ${revision} 4.0.0 Transaction Outbox Quarkus jar transactionoutbox-quarkus A safe implementation of the transactional outbox pattern for Java (Quarkus extension library) 3.31.4 com.gruelbox transactionoutbox-core ${project.version} jakarta.enterprise jakarta.enterprise.cdi-api jakarta.transaction jakarta.transaction-api io.quarkus quarkus-junit test io.quarkus quarkus-resteasy test io.quarkus quarkus-arc test io.quarkus quarkus-jdbc-h2 test io.quarkus quarkus-agroal test io.quarkus quarkus-undertow test io.quarkus.platform quarkus-bom ${quarkus.version} pom import ================================================ FILE: transactionoutbox-quarkus/src/main/java/com/gruelbox/transactionoutbox/quarkus/CdiInstantiator.java ================================================ package com.gruelbox.transactionoutbox.quarkus; import com.gruelbox.transactionoutbox.spi.AbstractFullyQualifiedNameInstantiator; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.inject.spi.CDI; @ApplicationScoped public class CdiInstantiator extends AbstractFullyQualifiedNameInstantiator { @SuppressWarnings("unused") public static CdiInstantiator create() { return new CdiInstantiator(); } private CdiInstantiator() {} @Override protected Object createInstance(Class clazz) { return CDI.current().select(clazz).get(); } } ================================================ FILE: transactionoutbox-quarkus/src/main/java/com/gruelbox/transactionoutbox/quarkus/QuarkusTransactionManager.java ================================================ package com.gruelbox.transactionoutbox.quarkus; import static com.gruelbox.transactionoutbox.spi.Utils.uncheck; import com.gruelbox.transactionoutbox.*; import com.gruelbox.transactionoutbox.spi.Utils; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import jakarta.transaction.Status; import jakarta.transaction.Synchronization; import jakarta.transaction.TransactionSynchronizationRegistry; import jakarta.transaction.Transactional; import jakarta.transaction.Transactional.TxType; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import javax.sql.DataSource; /** Transaction manager which uses cdi and quarkus. */ @ApplicationScoped public class QuarkusTransactionManager implements ThreadLocalContextTransactionManager { private final CdiTransaction transactionInstance = new CdiTransaction(); private final DataSource datasource; private final TransactionSynchronizationRegistry tsr; @Inject public QuarkusTransactionManager(DataSource datasource, TransactionSynchronizationRegistry tsr) { this.datasource = datasource; this.tsr = tsr; } @Override @Transactional(value = TxType.REQUIRES_NEW) public void inTransaction(Runnable runnable) { uncheck(() -> inTransactionReturnsThrows(ThrowingTransactionalSupplier.fromRunnable(runnable))); } @Override @Transactional(value = TxType.REQUIRES_NEW) public void inTransaction(TransactionalWork work) { uncheck(() -> inTransactionReturnsThrows(ThrowingTransactionalSupplier.fromWork(work))); } @Override @Transactional(value = TxType.REQUIRES_NEW) public T inTransactionReturnsThrows( ThrowingTransactionalSupplier work) throws E { return work.doWork(transactionInstance); } @Override public T requireTransactionReturns( ThrowingTransactionalSupplier work) throws E, NoTransactionActiveException { if (tsr.getTransactionStatus() != Status.STATUS_ACTIVE) { throw new NoTransactionActiveException(); } return work.doWork(transactionInstance); } private final class CdiTransaction implements Transaction { public Connection connection() { try { return datasource.getConnection(); } catch (SQLException e) { throw new RuntimeException(e); } } @Override public PreparedStatement prepareBatchStatement(String sql) { BatchCountingStatement preparedStatement = Utils.uncheckedly( () -> BatchCountingStatementHandler.countBatches(connection().prepareStatement(sql))); tsr.registerInterposedSynchronization( new Synchronization() { @Override public void beforeCompletion() { if (preparedStatement.getBatchCount() != 0) { Utils.uncheck(preparedStatement::executeBatch); } } @Override public void afterCompletion(int status) { Utils.safelyClose(preparedStatement); } }); return preparedStatement; } @Override public void addPostCommitHook(Runnable runnable) { tsr.registerInterposedSynchronization( new Synchronization() { @Override public void beforeCompletion() {} @Override public void afterCompletion(int status) { runnable.run(); } }); } } private interface BatchCountingStatement extends PreparedStatement { int getBatchCount(); } private static final class BatchCountingStatementHandler implements InvocationHandler { private final PreparedStatement delegate; private int count = 0; private BatchCountingStatementHandler(PreparedStatement delegate) { this.delegate = delegate; } static BatchCountingStatement countBatches(PreparedStatement delegate) { return (BatchCountingStatement) Proxy.newProxyInstance( BatchCountingStatementHandler.class.getClassLoader(), new Class[] {BatchCountingStatement.class}, new BatchCountingStatementHandler(delegate)); } public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { if ("getBatchCount".equals(method.getName())) { return count; } try { return method.invoke(delegate, args); } finally { if ("addBatch".equals(method.getName())) { ++count; } } } } } ================================================ FILE: transactionoutbox-quarkus/src/test/java/com/gruelbox/transactionoutbox/quarkus/acceptance/ApplicationConfig.java ================================================ package com.gruelbox.transactionoutbox.quarkus.acceptance; import com.gruelbox.transactionoutbox.*; import com.gruelbox.transactionoutbox.quarkus.CdiInstantiator; import com.gruelbox.transactionoutbox.quarkus.QuarkusTransactionManager; import jakarta.enterprise.inject.Produces; import jakarta.ws.rs.core.Application; import java.util.HashSet; import java.util.Set; public class ApplicationConfig extends Application { @Override public Set> getClasses() { final Set> classes = new HashSet>(); classes.add(BusinessService.class); return classes; } @Produces public TransactionOutbox transactionOutbox( QuarkusTransactionManager transactionManager, RemoteCallService testProxy) { return TransactionOutbox.builder() .instantiator(CdiInstantiator.create()) .blockAfterAttempts(1) .listener( new TransactionOutboxListener() { @Override public void blocked(TransactionOutboxEntry entry, Throwable cause) { block(testProxy); } }) .transactionManager(transactionManager) .persistor(Persistor.forDialect(Dialect.H2)) .build(); } private void block(RemoteCallService testProxy) { testProxy.block(); } } ================================================ FILE: transactionoutbox-quarkus/src/test/java/com/gruelbox/transactionoutbox/quarkus/acceptance/BusinessService.java ================================================ package com.gruelbox.transactionoutbox.quarkus.acceptance; import com.gruelbox.transactionoutbox.TransactionOutbox; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import jakarta.transaction.Transactional; @ApplicationScoped public class BusinessService { private final DaoImpl dao; private final TransactionOutbox outbox; @Inject public BusinessService(DaoImpl dao, TransactionOutbox outbox) { this.dao = dao; this.outbox = outbox; } @Transactional public void writeSomeThingAndRemoteCall(String value, boolean throwException) { dao.writeSomethingIntoDatabase(value); RemoteCallService proxy = outbox.schedule(RemoteCallService.class); proxy.callRemote(throwException); } } ================================================ FILE: transactionoutbox-quarkus/src/test/java/com/gruelbox/transactionoutbox/quarkus/acceptance/BusinessServiceTest.java ================================================ package com.gruelbox.transactionoutbox.quarkus.acceptance; import io.quarkus.test.junit.QuarkusTest; import jakarta.inject.Inject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @QuarkusTest public class BusinessServiceTest { @Inject private BusinessService res; @Inject private RemoteCallService remoteCall; @Inject private DaoImpl dao; @BeforeEach void purgeDatabase() { dao.purge(); remoteCall.setCalled(false); remoteCall.setBlocked(false); } @Test void writeOperationAndRemoteCallOK() throws Exception { Assertions.assertFalse(remoteCall.isCalled()); res.writeSomeThingAndRemoteCall("toto", false); Thread.sleep(1000); Assertions.assertTrue(remoteCall.isCalled()); Assertions.assertFalse(dao.getFromDatabase().isEmpty()); } @Test void writeOperationOkButRemoteCallErrorShouldBlockRemoteCall() throws Exception { Assertions.assertFalse(remoteCall.isCalled()); res.writeSomeThingAndRemoteCall("toto", true); Thread.sleep(1000); Assertions.assertFalse(remoteCall.isCalled()); Assertions.assertFalse(dao.getFromDatabase().isEmpty()); Assertions.assertTrue(remoteCall.isBlocked()); } @Test void transactionRollbackSoRemoteCallShouldNotBeMade() throws Exception { Assertions.assertFalse(remoteCall.isCalled()); try { res.writeSomeThingAndRemoteCall("error", false); Assertions.fail("Should not happen"); } catch (RuntimeException e) { Assertions.assertEquals("Persistence error", e.getMessage()); } Thread.sleep(1000); Assertions.assertFalse(remoteCall.isCalled()); Assertions.assertTrue(dao.getFromDatabase().isEmpty()); Assertions.assertFalse(remoteCall.isBlocked()); } } ================================================ FILE: transactionoutbox-quarkus/src/test/java/com/gruelbox/transactionoutbox/quarkus/acceptance/DaoImpl.java ================================================ package com.gruelbox.transactionoutbox.quarkus.acceptance; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.sql.*; import java.util.ArrayList; import java.util.List; import javax.sql.DataSource; @ApplicationScoped public class DaoImpl { @Inject DataSource defaultDataSource; @SuppressWarnings("UnusedReturnValue") public int writeSomethingIntoDatabase(String something) { if ("error".equals(something)) { throw new RuntimeException("Persistence error"); } String insertQuery = "insert into toto values (?);"; try (Connection connexion = defaultDataSource.getConnection(); PreparedStatement statement = connexion.prepareStatement(insertQuery)) { statement.setString(1, something); return statement.executeUpdate(); } catch (SQLException e) { throw new RuntimeException(e); } } public List getFromDatabase() { List values = new ArrayList<>(); try (Connection connexion = defaultDataSource.getConnection(); Statement statement = connexion.createStatement()) { ResultSet resultSet = statement.executeQuery("select * from toto;"); while (resultSet.next()) { values.add(resultSet.getString(1)); } return values; } catch (SQLException e) { throw new RuntimeException(e); } } @SuppressWarnings("UnusedReturnValue") public int purge() { try (Connection connexion = defaultDataSource.getConnection(); Statement statement = connexion.createStatement()) { return statement.executeUpdate("delete from toto;"); } catch (SQLException e) { throw new RuntimeException(e); } } } ================================================ FILE: transactionoutbox-quarkus/src/test/java/com/gruelbox/transactionoutbox/quarkus/acceptance/RemoteCallService.java ================================================ package com.gruelbox.transactionoutbox.quarkus.acceptance; import jakarta.enterprise.context.ApplicationScoped; @ApplicationScoped public class RemoteCallService { private boolean called; private boolean blocked; public void callRemote(boolean throwException) { if (throwException) { throw new RuntimeException("Thrown on purpose"); } called = true; } public boolean isCalled() { return called; } public void setCalled(boolean called) { this.called = called; } public void block() { this.blocked = true; } public boolean isBlocked() { return blocked; } public void setBlocked(boolean blocked) { this.blocked = blocked; } } ================================================ FILE: transactionoutbox-quarkus/src/test/resources/application.properties ================================================ quarkus.application.name=transaction-outbox quarkus.http.port = 8082 quarkus.datasource.db-kind = h2 quarkus.datasource.username = test quarkus.datasource.password = test quarkus.datasource.jdbc.url = jdbc:h2:mem:test;INIT=RUNSCRIPT FROM 'src/test/resources/db/create.sql' quarkus.test.flat-class-path=true %test.quarkus.log.level=INFO ================================================ FILE: transactionoutbox-quarkus/src/test/resources/db/create.sql ================================================ CREATE TABLE IF NOT EXISTS toto (toto VARCHAR(50) NOT NULL); ================================================ FILE: transactionoutbox-spring/README.md ================================================ # transaction-outbox-spring [![Spring on Maven Central](https://maven-badges.herokuapp.com/maven-central/com.gruelbox/transactionoutbox-spring/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.gruelbox/transactionoutbox-spring) [![Spring Javadoc](https://www.javadoc.io/badge/com.gruelbox/transactionoutbox-spring.svg?color=blue)](https://www.javadoc.io/doc/com.gruelbox/transactionoutbox-spring) [![Latest snapshot](https://img.shields.io/github/v/tag/gruelbox/transaction-outbox?label=snapshot&sort=semver)](#development-snapshots) Extension for [transaction-outbox-core](../README.md) which integrates Spring's DI and/or transaction management. I don't actually use Spring in production, so this is more presented as an example at the moment. Doubtless I've missed a lot of nuances about the flexibility of Spring. Pull requests very welcome. ## Installation ### Stable releases The latest stable release is available from Maven Central. #### Maven ```xml com.gruelbox transactionoutbox-spring 7.0.707 ``` #### Gradle ```groovy implementation 'com.gruelbox:transactionoutbox-spring:7.0.707' ``` ### Development snapshots Maven Central is updated regularly. Alternatively, if you want to stay at the bleeding edge, you can use continuously-delivered releases from [Github Package Repository](https://github.com/gruelbox/transaction-outbox/packages). These can be used from production builds since they will never be deleted. #### Maven ```xml github-transaction-outbox Gruelbox Github Repository https://maven.pkg.github.com/gruelbox/transaction-outbox ``` You will need to authenticate with Github to use Github Package Repository. Create a personal access token in [your GitHub settings](https://github.com/settings/tokens). It only needs **read:package** permissions. Then add something like the following in your Maven `settings.xml`: ```xml github-transaction-outbox ${env.GITHUB_USERNAME} ${env.GITHUB_TOKEN} ``` The above example uses environment variables, allowing you to keep the credentials out of source control, but you can hard-code them if you know what you're doing. ## Example An example application can be found here: https://github.com/gruelbox/transaction-outbox/tree/better-spring-example/transactionoutbox-spring/src/test. ## Configuration Create your `TransactionOutbox` as a bean: ```java @Bean @Lazy public TransactionOutbox transactionOutbox(SpringTransactionManager springTransactionManager, SpringInstantiator springInstantiator) { return TransactionOutbox.builder() .instantiator(springInstantiator) .transactionManager(springTransactionManager) .persistor(Persistor.forDialect(Dialect.H2)) .build(); ``` You can mix-and-match `SpringInstantiator` ans `SpringTransactionManager` with other implementations in hybrid frameworks. ## Usage ```java @Transactional public void doStuff() { customerRepository.save(new Customer(1L, "Martin", "Carthy")); customerRepository.save(new Customer(2L, "Dave", "Pegg")); outbox.get().schedule(getClass()).publishCustomerCreatedEvent(1L); outbox.get().schedule(getClass()).publishCustomerCreatedEvent(2L); } void publishCustomerCreatedEvent(long id) { // Remote call here } ``` Notice that with a DI framework like Spring in play, you can **self-invoke** on `getClass()` - invoke a method on the same class that's scheduling it. ================================================ FILE: transactionoutbox-spring/pom.xml ================================================ transactionoutbox-parent com.gruelbox ${revision} 4.0.0 Transaction Outbox Spring jar transactionoutbox-spring A safe implementation of the transactional outbox pattern for Java (Spring extension library) 17 17 4.0.3 7.0.4 com.gruelbox transactionoutbox-core ${project.version} org.springframework spring-tx ${spring.version} provided org.springframework spring-jdbc ${spring.version} provided org.springframework spring-context ${spring.version} provided org.projectlombok lombok com.gruelbox transactionoutbox-jackson test ${project.version} ch.qos.logback logback-classic ch.qos.logback logback-core com.h2database h2 com.zaxxer HikariCP org.springframework.boot spring-boot-jackson2 ${spring.boot.version} org.springframework.boot spring-boot-starter-data-jpa ${spring.boot.version} test org.springframework.boot spring-boot-starter-web ${spring.boot.version} test org.springframework.boot spring-boot-starter-test ${spring.boot.version} test org.junit.jupiter junit-jupiter-engine org.junit.jupiter junit-jupiter-api org.junit.jupiter junit-jupiter-params org.awaitility awaitility 4.3.0 test ================================================ FILE: transactionoutbox-spring/src/main/java/com/gruelbox/transactionoutbox/spring/SpringInstantiator.java ================================================ package com.gruelbox.transactionoutbox.spring; import com.gruelbox.transactionoutbox.Instantiator; import java.util.Arrays; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Service; /** * Instantiator that uses the spring {@link ApplicationContext} to source objects. It requires that * classes scheduled have a unique name in the context, so doesn't often play well with proxies and * other auto-generated code such as repositories based on {@code CrudRepository}. */ @Service public class SpringInstantiator implements Instantiator { private final ApplicationContext applicationContext; @Autowired public SpringInstantiator(ApplicationContext applicationContext) { this.applicationContext = applicationContext; } @Override public String getName(Class clazz) { String[] beanNames = applicationContext.getBeanNamesForType(clazz); if (beanNames.length > 1) { throw new IllegalArgumentException( "Type " + clazz.getName() + " exists under multiple names in the context: " + Arrays.toString(beanNames) + ". Use a unique type."); } if (beanNames.length == 0) { throw new IllegalArgumentException( "Type " + clazz.getName() + " not available in the context"); } return beanNames[0]; } @Override public Object getInstance(String name) { return applicationContext.getBean(name); } } ================================================ FILE: transactionoutbox-spring/src/main/java/com/gruelbox/transactionoutbox/spring/SpringTransactionManager.java ================================================ package com.gruelbox.transactionoutbox.spring; import static com.gruelbox.transactionoutbox.spi.Utils.uncheck; import static com.gruelbox.transactionoutbox.spi.Utils.uncheckedly; import com.gruelbox.transactionoutbox.*; import com.gruelbox.transactionoutbox.spi.Utils; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.sql.Connection; import java.sql.PreparedStatement; import javax.sql.DataSource; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.datasource.DataSourceUtils; import org.springframework.stereotype.Service; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.support.TransactionSynchronization; import org.springframework.transaction.support.TransactionSynchronizationManager; import org.springframework.transaction.support.TransactionTemplate; /** Transaction manager which uses spring-tx and Hibernate. */ @Slf4j @Service public class SpringTransactionManager implements ThreadLocalContextTransactionManager { private final SpringTransaction transactionInstance = new SpringTransaction(); private final PlatformTransactionManager platformTransactionManager; private final DataSource dataSource; @Autowired public SpringTransactionManager( PlatformTransactionManager platformTransactionManager, DataSource dataSource) { this.platformTransactionManager = platformTransactionManager; this.dataSource = dataSource; } @Override public void inTransaction(Runnable runnable) { uncheck(() -> inTransactionReturnsThrows(ThrowingTransactionalSupplier.fromRunnable(runnable))); } @Override public void inTransaction(TransactionalWork work) { uncheck(() -> inTransactionReturnsThrows(ThrowingTransactionalSupplier.fromWork(work))); } @Override public T inTransactionReturns(TransactionalSupplier supplier) { return uncheckedly( () -> inTransactionReturnsThrows(ThrowingTransactionalSupplier.fromSupplier(supplier))); } @Override public void inTransactionThrows(ThrowingTransactionalWork work) throws E { inTransactionReturnsThrows(ThrowingTransactionalSupplier.fromWork(work)); } @Override public T inTransactionReturnsThrows( ThrowingTransactionalSupplier work) throws E { TransactionTemplate transactionTemplate = new TransactionTemplate(platformTransactionManager); transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); try { return transactionTemplate.execute( status -> { try { return work.doWork(transactionInstance); } catch (Exception e) { throw new UncheckedException(e); } }); } catch (UncheckedException e) { @SuppressWarnings("unchecked") E cause = (E) e.getCause(); throw cause; } } @Override public T requireTransactionReturns( ThrowingTransactionalSupplier work) throws E, NoTransactionActiveException { if (!TransactionSynchronizationManager.isActualTransactionActive()) { throw new NoTransactionActiveException(); } return work.doWork(transactionInstance); } private final class SpringTransaction implements Transaction { @Override public Connection connection() { return DataSourceUtils.getConnection(dataSource); } @Override public PreparedStatement prepareBatchStatement(String sql) { BatchCountingStatement preparedStatement = Utils.uncheckedly( () -> BatchCountingStatementHandler.countBatches(connection().prepareStatement(sql))); TransactionSynchronizationManager.registerSynchronization( new TransactionSynchronization() { @Override public void beforeCommit(boolean readOnly) { if (preparedStatement.getBatchCount() != 0) { log.debug("Flushing batches"); Utils.uncheck(preparedStatement::executeBatch); } } @Override public void afterCompletion(int status) { Utils.safelyClose(preparedStatement); } }); return preparedStatement; } @Override public void addPostCommitHook(Runnable runnable) { TransactionSynchronizationManager.registerSynchronization( new TransactionSynchronization() { @Override public void afterCommit() { runnable.run(); } }); } } private interface BatchCountingStatement extends PreparedStatement { int getBatchCount(); } private static final class BatchCountingStatementHandler implements InvocationHandler { private final PreparedStatement delegate; private int count = 0; private BatchCountingStatementHandler(PreparedStatement delegate) { this.delegate = delegate; } static BatchCountingStatement countBatches(PreparedStatement delegate) { return (BatchCountingStatement) Proxy.newProxyInstance( BatchCountingStatementHandler.class.getClassLoader(), new Class[] {BatchCountingStatement.class}, new BatchCountingStatementHandler(delegate)); } public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { if ("getBatchCount".equals(method.getName())) { return count; } try { return method.invoke(delegate, args); } catch (InvocationTargetException e) { throw e.getCause(); } finally { if ("addBatch".equals(method.getName())) { ++count; } } } } } ================================================ FILE: transactionoutbox-spring/src/main/java/com/gruelbox/transactionoutbox/spring/SpringTransactionOutboxConfiguration.java ================================================ package com.gruelbox.transactionoutbox.spring; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; /** * @deprecated Just {@code @Import} the components you need. */ @Configuration @Deprecated(forRemoval = true) @Import({SpringTransactionManager.class, SpringInstantiator.class}) public class SpringTransactionOutboxConfiguration {} ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/SpringTransactionManagerTest.java ================================================ package com.gruelbox.transactionoutbox.spring; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.verify; import com.gruelbox.transactionoutbox.UncheckedException; import java.sql.SQLException; import javax.sql.DataSource; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.TransactionException; import org.springframework.transaction.TransactionStatus; @ExtendWith(MockitoExtension.class) class SpringTransactionManagerTest { @Mock private PlatformTransactionManager platformTransactionManager; @Mock private DataSource dataSource; @Mock private TransactionStatus transactionStatus; @Mock private SpringTransactionManager springTransactionManager; @BeforeEach void setUp() { Mockito.when(platformTransactionManager.getTransaction(Mockito.any())) .thenReturn(transactionStatus); springTransactionManager = new SpringTransactionManager(platformTransactionManager, dataSource); } private static class MyRuntimeException extends RuntimeException {} private static class MyCheckedException extends Exception {} private static class MyUncheckedException extends UncheckedException { public MyUncheckedException(Throwable cause) { super(cause); } } private static class MySpringTransactionException extends TransactionException { public MySpringTransactionException(String msg, Throwable cause) { super(msg, cause); } } private static class MySqlException extends SQLException {} @Test void shouldWorkInNewTransactionAndCommit() { springTransactionManager.inTransactionReturnsThrows(transaction -> true); verify(platformTransactionManager) .getTransaction( ArgumentMatchers.assertArg( trxDef -> { Assertions.assertThat(trxDef).isNotNull(); Assertions.assertThat(trxDef.getPropagationBehavior()) .isEqualTo(TransactionDefinition.PROPAGATION_REQUIRES_NEW); })); verify(platformTransactionManager).commit(same(transactionStatus)); } @Test void shouldRollbackOnFailure() { assertThrows( RuntimeException.class, () -> { springTransactionManager.inTransactionReturnsThrows( transaction -> { throw new RuntimeException(); }); }); verify(platformTransactionManager) .getTransaction( ArgumentMatchers.assertArg( trxDef -> { Assertions.assertThat(trxDef).isNotNull(); Assertions.assertThat(trxDef.getPropagationBehavior()) .isEqualTo(TransactionDefinition.PROPAGATION_REQUIRES_NEW); })); verify(platformTransactionManager).rollback(same(transactionStatus)); } @Test void shouldPreserveRuntimeException() { assertThrows( MyRuntimeException.class, () -> { springTransactionManager.inTransactionReturnsThrows( transaction -> { throw new MyRuntimeException(); }); }); } @Test void shouldPreserveCheckedException() { assertThrows( MyCheckedException.class, () -> { springTransactionManager.inTransactionReturnsThrows( transaction -> { throw new MyCheckedException(); }); }); } @Test void shouldPreserveUncheckedException() { assertThrows( MyUncheckedException.class, () -> { springTransactionManager.inTransactionReturnsThrows( transaction -> { throw new MyUncheckedException(new RuntimeException()); }); }); } @Test void shouldPreserveSpringTransactionException() { assertThrows( MySpringTransactionException.class, () -> { springTransactionManager.inTransactionReturnsThrows( transaction -> { throw new MySpringTransactionException("expected failure", new RuntimeException()); }); }); } @Test void shouldPreserveSqlException() { assertThrows( MySqlException.class, () -> { springTransactionManager.inTransactionReturnsThrows( transaction -> { throw new MySqlException(); }); }); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/EventuallyConsistentController.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources; import static org.springframework.http.HttpStatus.NOT_FOUND; import com.gruelbox.transactionoutbox.TransactionOutbox; import com.gruelbox.transactionoutbox.spring.example.multipledatasources.computer.Computer; import com.gruelbox.transactionoutbox.spring.example.multipledatasources.computer.ComputerExternalQueueService; import com.gruelbox.transactionoutbox.spring.example.multipledatasources.computer.ComputerRepository; import com.gruelbox.transactionoutbox.spring.example.multipledatasources.employee.Employee; import com.gruelbox.transactionoutbox.spring.example.multipledatasources.employee.EmployeeExternalQueueService; import com.gruelbox.transactionoutbox.spring.example.multipledatasources.employee.EmployeeRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ResponseStatusException; @SuppressWarnings("unused") @RestController public class EventuallyConsistentController { @Autowired private ComputerRepository computerRepository; @Autowired private TransactionOutbox computerTransactionOutbox; @Autowired private EmployeeRepository employeeRepository; @Autowired private TransactionOutbox employeeTransactionOutbox; @SuppressWarnings("SameReturnValue") @PostMapping("/computer") @Transactional(transactionManager = "computerTransactionManager") public void createComputer( @RequestBody Computer computer, @RequestParam(name = "ordered", required = false) Boolean ordered) { computerRepository.save(computer); if (ordered != null && ordered) { computerTransactionOutbox .with() .ordered("justonetopic") .schedule(ComputerExternalQueueService.class) .sendComputerCreatedEvent(computer); } else { computerTransactionOutbox .schedule(ComputerExternalQueueService.class) .sendComputerCreatedEvent(computer); } } @GetMapping("/computer/{id}") public Computer getComputer(@PathVariable long id) { return computerRepository .findById(id) .orElseThrow(() -> new ResponseStatusException(NOT_FOUND)); } @SuppressWarnings("SameReturnValue") @PostMapping("/employee") @Transactional(transactionManager = "employeeTransactionManager") public void createEmployee( @RequestBody Employee employee, @RequestParam(name = "ordered", required = false) Boolean ordered) { employeeRepository.save(employee); if (ordered != null && ordered) { employeeTransactionOutbox .with() .ordered("justonetopic") .schedule(EmployeeExternalQueueService.class) .sendEmployeeCreatedEvent(employee); } else { employeeTransactionOutbox .schedule(EmployeeExternalQueueService.class) .sendEmployeeCreatedEvent(employee); } } @GetMapping("/employee/{id}") public Employee getEmployee(@PathVariable long id) { return employeeRepository .findById(id) .orElseThrow(() -> new ResponseStatusException(NOT_FOUND)); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/ExternalsConfiguration.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources; import com.gruelbox.transactionoutbox.spring.SpringInstantiator; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; @Configuration @Import({SpringInstantiator.class}) class ExternalsConfiguration {} ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/MultipleDataSourcesTest.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertTrue; import com.gruelbox.transactionoutbox.spring.example.multipledatasources.computer.Computer; import com.gruelbox.transactionoutbox.spring.example.multipledatasources.computer.Computer.Type; import com.gruelbox.transactionoutbox.spring.example.multipledatasources.computer.ComputerExternalQueueService; import com.gruelbox.transactionoutbox.spring.example.multipledatasources.employee.Employee; import com.gruelbox.transactionoutbox.spring.example.multipledatasources.employee.EmployeeExternalQueueService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.web.server.LocalServerPort; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.web.client.RestClient; @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) class MultipleDataSourcesTest { @LocalServerPort private int port; private RestClient restClient; @Autowired private JdbcTemplate employeeJdbcTemplate; @Autowired private JdbcTemplate computerJdbcTemplate; @Autowired private EmployeeExternalQueueService employeeExternalQueueService; @Autowired private ComputerExternalQueueService computerExternalQueueService; @BeforeEach void setUp() { this.restClient = RestClient.builder().baseUrl("http://localhost:" + port).build(); employeeExternalQueueService.clear(); computerExternalQueueService.clear(); } @Test void testCheckNormalEmployees() { var joe = new Employee(1L, "Joe", "Strummer"); var dave = new Employee(2L, "Dave", "Grohl"); var neil = new Employee(3L, "Neil", "Diamond"); var tupac = new Employee(4L, "Tupac", "Shakur"); var jeff = new Employee(5L, "Jeff", "Mills"); assertTrue( restClient .post() .uri("/employee") .body(joe) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/employee") .body(dave) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/employee") .body(neil) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/employee") .body(tupac) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/employee") .body(jeff) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); employeeJdbcTemplate.execute( "UPDATE txno_outbox SET invocation='non-deserializable invocation' WHERE invocation LIKE '%" + neil.getLastName() + "%'"); await() .atMost(10, SECONDS) .pollDelay(1, SECONDS) .untilAsserted( () -> assertThat(employeeExternalQueueService.getSent()) .containsExactlyInAnyOrder(joe, dave, tupac, jeff)); } @Test void testCheckNormalComputers() throws InterruptedException { var computerPc1 = new Computer(1L, "pc-001", Type.DESKTOP); var computerPc2 = new Computer(2L, "pc-002", Type.LAPTOP); var computerPc3 = new Computer(3L, "pc-003", Type.LAPTOP); var computerWebserver1 = new Computer(4L, "webserver-001", Type.SERVER); var computerWebserver2 = new Computer(5L, "webserver-002", Type.SERVER); assertTrue( restClient .post() .uri("/computer") .body(computerPc1) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/computer") .body(computerPc2) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/computer") .body(computerPc3) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/computer") .body(computerWebserver1) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/computer") .body(computerWebserver2) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); computerJdbcTemplate.execute( "UPDATE txno_outbox SET invocation='non-deserializable invocation' WHERE invocation LIKE '%" + computerPc3.getName() + "%'"); await() .atMost(10, SECONDS) .pollDelay(1, SECONDS) .untilAsserted( () -> assertThat(computerExternalQueueService.getSent()) .containsExactlyInAnyOrder( computerPc1, computerPc2, computerWebserver1, computerWebserver2)); } @Test void testCheckOrderedEmployees() { var joe = new Employee(1L, "Joe", "Strummer"); var dave = new Employee(2L, "Dave", "Grohl"); var neil = new Employee(3L, "Neil", "Diamond"); var tupac = new Employee(4L, "Tupac", "Shakur"); var jeff = new Employee(5L, "Jeff", "Mills"); assertTrue( restClient .post() .uri("/employee?ordered=true") .body(joe) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/employee?ordered=true") .body(dave) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/employee?ordered=true") .body(neil) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/employee?ordered=true") .body(tupac) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/employee?ordered=true") .body(jeff) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); employeeJdbcTemplate.execute( "UPDATE txno_outbox SET invocation='non-deserializable invocation' WHERE invocation LIKE '%" + neil.getLastName() + "%'"); await() .atMost(10, SECONDS) .pollDelay(1, SECONDS) .untilAsserted( () -> assertThat(employeeExternalQueueService.getSent()).containsExactly(joe, dave)); } @Test void testCheckOrderedComputers() { var computerPc1 = new Computer(1L, "pc-001", Type.DESKTOP); var computerPc2 = new Computer(2L, "pc-002", Type.LAPTOP); var computerPc3 = new Computer(3L, "pc-003", Type.LAPTOP); var computerWebserver1 = new Computer(4L, "webserver-001", Type.SERVER); var computerWebserver2 = new Computer(5L, "webserver-002", Type.SERVER); assertTrue( restClient .post() .uri("/computer?ordered=true") .body(computerPc1) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/computer?ordered=true") .body(computerPc2) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/computer?ordered=true") .body(computerPc3) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/computer?ordered=true") .body(computerWebserver1) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/computer?ordered=true") .body(computerWebserver2) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); employeeJdbcTemplate.execute( "UPDATE txno_outbox SET invocation='non-deserializable invocation' WHERE invocation LIKE '%" + computerPc3.getName() + "%'"); await() .atMost(10, SECONDS) .pollDelay(1, SECONDS) .untilAsserted( () -> assertThat(computerExternalQueueService.getSent()) .containsExactly(computerPc1, computerPc2)); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/TransactionOutboxBackgroundProcessor.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources; import com.gruelbox.transactionoutbox.TransactionOutbox; import java.util.Collection; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; /** * Simple implementation of a background processor for {@link TransactionOutbox}. You don't need to * use this if you need different semantics, but this is a good start for most purposes. */ @Component @Slf4j @RequiredArgsConstructor(onConstructor_ = {@Autowired}) class TransactionOutboxBackgroundProcessor { private final Collection outboxes; @Scheduled(fixedRateString = "${outbox.repeatEvery}") void poll() { outboxes.forEach( outbox -> { try { do { log.info("Flushing"); } while (outbox.flush()); } catch (Exception e) { log.error("Error flushing transaction outbox. Pausing", e); } }); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/TransactionOutboxProperties.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources; import java.time.Duration; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; @Configuration @ConfigurationProperties("outbox") @Data class TransactionOutboxProperties { private Duration repeatEvery; private boolean useJackson; private Duration attemptFrequency; private int blockAfterAttempts; } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/TransactionOutboxSpringMultipleDatasourcesDemoApplication.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources; import com.fasterxml.jackson.databind.ObjectMapper; import com.gruelbox.transactionoutbox.DefaultPersistor; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.Persistor; import com.gruelbox.transactionoutbox.TransactionOutbox; import com.gruelbox.transactionoutbox.jackson.JacksonInvocationSerializer; import com.gruelbox.transactionoutbox.spring.SpringInstantiator; import com.gruelbox.transactionoutbox.spring.SpringTransactionManager; import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Lazy; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.transaction.PlatformTransactionManager; @SpringBootApplication @EnableScheduling public class TransactionOutboxSpringMultipleDatasourcesDemoApplication { public static void main(String[] args) { SpringApplication.run(TransactionOutboxSpringMultipleDatasourcesDemoApplication.class, args); } @Bean @Lazy Persistor persistor(TransactionOutboxProperties properties, ObjectMapper objectMapper) { if (properties.isUseJackson()) { return DefaultPersistor.builder() .serializer(JacksonInvocationSerializer.builder().mapper(objectMapper).build()) .dialect(Dialect.H2) .build(); } else { return Persistor.forDialect(Dialect.H2); } } @Bean @Lazy TransactionOutbox computerTransactionOutbox( SpringInstantiator instantiator, @Qualifier("computerSpringTransactionManager") SpringTransactionManager transactionManager, TransactionOutboxProperties properties, Persistor persistor) { return TransactionOutbox.builder() .instantiator(instantiator) .transactionManager(transactionManager) .persistor(persistor) .attemptFrequency(properties.getAttemptFrequency()) .blockAfterAttempts(properties.getBlockAfterAttempts()) .build(); } @Bean @Lazy TransactionOutbox employeeTransactionOutbox( SpringInstantiator instantiator, @Qualifier("employeeSpringTransactionManager") SpringTransactionManager transactionManager, TransactionOutboxProperties properties, Persistor persistor) { return TransactionOutbox.builder() .instantiator(instantiator) .transactionManager(transactionManager) .persistor(persistor) .attemptFrequency(properties.getAttemptFrequency()) .blockAfterAttempts(properties.getBlockAfterAttempts()) .build(); } @Bean public SpringTransactionManager computerSpringTransactionManager( @Qualifier("computerTransactionManager") PlatformTransactionManager transactionManager, @Qualifier("computerDataSource") DataSource dataSource) { return new SpringTransactionManager(transactionManager, dataSource); } @Bean public SpringTransactionManager employeeSpringTransactionManager( @Qualifier("employeeTransactionManager") PlatformTransactionManager transactionManager, @Qualifier("employeeDataSource") DataSource dataSource) { return new SpringTransactionManager(transactionManager, dataSource); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/computer/Computer.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources.computer; import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.Id; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Entity @Data @NoArgsConstructor @AllArgsConstructor public class Computer { public enum Type { LAPTOP, SERVER, DESKTOP; } @Id private Long id; @Column private String name; @Column private Type type; } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/computer/ComputerExternalQueueService.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources.computer; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import lombok.Getter; import org.springframework.stereotype.Service; @Getter @Service public class ComputerExternalQueueService { private final Set attempted = new HashSet<>(); private final List sent = new CopyOnWriteArrayList<>(); public void sendComputerCreatedEvent(Computer computer) { if (attempted.add(computer.getId())) { throw new RuntimeException("Temporary failure, try again"); } sent.add(computer); } public void clear() { attempted.clear(); sent.clear(); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/computer/ComputerRepository.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources.computer; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Repository; @Repository public interface ComputerRepository extends CrudRepository {} ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/computer/ComputersDbConfiguration.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources.computer; import jakarta.persistence.EntityManagerFactory; import java.util.Map; import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.jdbc.autoconfigure.DataSourceProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; import org.springframework.transaction.PlatformTransactionManager; @Configuration @EnableJpaRepositories( basePackageClasses = Computer.class, entityManagerFactoryRef = "computerEntityManager", transactionManagerRef = "computerTransactionManager") public class ComputersDbConfiguration { @Bean public DataSourceProperties computerDataSourceProperties() { DataSourceProperties properties = new DataSourceProperties(); properties.setDriverClassName(org.h2.Driver.class.getName()); properties.setUrl("jdbc:h2:mem:computer"); properties.setUsername("computerUser"); properties.setPassword("computerPassword"); return properties; } @Bean public DataSource computerDataSource() { return computerDataSourceProperties().initializeDataSourceBuilder().build(); } @Bean public JdbcTemplate computerJdbcTemplate() { return new JdbcTemplate(computerDataSource()); } @Bean public LocalContainerEntityManagerFactoryBean computerEntityManager() { LocalContainerEntityManagerFactoryBean emf = new LocalContainerEntityManagerFactoryBean(); emf.setDataSource(computerDataSource()); emf.setPackagesToScan(Computer.class.getPackage().getName()); emf.setJpaVendorAdapter(new HibernateJpaVendorAdapter()); emf.setJpaPropertyMap( Map.of( "hibernate.hbm2ddl.auto", "update", "hibernate.show_sql", "true")); emf.setPersistenceUnitName("computer"); return emf; } @Bean public PlatformTransactionManager computerTransactionManager( @Qualifier("computerEntityManager") EntityManagerFactory entityManagerFactory) { return new JpaTransactionManager(entityManagerFactory); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/employee/Employee.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources.employee; import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.Id; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Entity @Data @NoArgsConstructor @AllArgsConstructor public class Employee { @Id private Long id; @Column private String firstName; @Column private String lastName; } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/employee/EmployeeExternalQueueService.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources.employee; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import lombok.Getter; import org.springframework.stereotype.Service; @Getter @Service public class EmployeeExternalQueueService { private final Set attempted = new HashSet<>(); private final List sent = new CopyOnWriteArrayList<>(); public void sendEmployeeCreatedEvent(Employee employee) { if (attempted.add(employee.getId())) { throw new RuntimeException("Temporary failure, try again"); } sent.add(employee); } public void clear() { attempted.clear(); sent.clear(); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/employee/EmployeeRepository.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources.employee; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Repository; @Repository public interface EmployeeRepository extends CrudRepository {} ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/multipledatasources/employee/EmployeesDbConfiguration.java ================================================ package com.gruelbox.transactionoutbox.spring.example.multipledatasources.employee; import jakarta.persistence.EntityManagerFactory; import java.util.Map; import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.jdbc.autoconfigure.DataSourceProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; import org.springframework.transaction.PlatformTransactionManager; @Configuration @EnableJpaRepositories( basePackageClasses = Employee.class, entityManagerFactoryRef = "employeeEntityManager", transactionManagerRef = "employeeTransactionManager") public class EmployeesDbConfiguration { @Bean @ConfigurationProperties("spring.datasource.employees") public DataSourceProperties employeeDataSourceProperties() { DataSourceProperties properties = new DataSourceProperties(); properties.setDriverClassName(org.h2.Driver.class.getName()); properties.setUrl("jdbc:h2:mem:employee"); properties.setUsername("employeeUser"); properties.setPassword("employeePassword"); return properties; } @Bean public DataSource employeeDataSource() { return employeeDataSourceProperties().initializeDataSourceBuilder().build(); } @Bean public JdbcTemplate employeeJdbcTemplate() { return new JdbcTemplate(employeeDataSource()); } @Bean public LocalContainerEntityManagerFactoryBean employeeEntityManager() { LocalContainerEntityManagerFactoryBean emf = new LocalContainerEntityManagerFactoryBean(); emf.setDataSource(employeeDataSource()); emf.setPackagesToScan(Employee.class.getPackage().getName()); emf.setJpaVendorAdapter(new HibernateJpaVendorAdapter()); emf.setJpaPropertyMap( Map.of( "hibernate.hbm2ddl.auto", "update", "hibernate.show_sql", "true")); emf.setPersistenceUnitName("employee"); return emf; } @Bean public PlatformTransactionManager employeeTransactionManager( @Qualifier("employeeEntityManager") EntityManagerFactory entityManagerFactory) { return new JpaTransactionManager(entityManagerFactory); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/simple/Customer.java ================================================ package com.gruelbox.transactionoutbox.spring.example.simple; import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.Id; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Entity @Data @NoArgsConstructor @AllArgsConstructor class Customer { @Id private Long id; @Column private String firstName; @Column private String lastName; } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/simple/CustomerRepository.java ================================================ package com.gruelbox.transactionoutbox.spring.example.simple; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Repository; @Repository interface CustomerRepository extends CrudRepository {} ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/simple/EventuallyConsistentController.java ================================================ package com.gruelbox.transactionoutbox.spring.example.simple; import static org.springframework.http.HttpStatus.NOT_FOUND; import com.gruelbox.transactionoutbox.TransactionOutbox; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.*; import org.springframework.web.server.ResponseStatusException; @SuppressWarnings("unused") @RestController class EventuallyConsistentController { @Autowired private CustomerRepository customerRepository; @Autowired private TransactionOutbox outbox; @SuppressWarnings("SameReturnValue") @PostMapping(path = "/customer") @Transactional public void createCustomer( @RequestBody Customer customer, @RequestParam(name = "ordered", required = false) Boolean ordered) { customerRepository.save(customer); if (ordered != null && ordered) { outbox .with() .ordered("justonetopic") .schedule(ExternalQueueService.class) .sendCustomerCreatedEvent(customer); } else { outbox.schedule(ExternalQueueService.class).sendCustomerCreatedEvent(customer); } } @GetMapping("/customer/{id}") public Customer getCustomer(@PathVariable long id) { return customerRepository .findById(id) .orElseThrow(() -> new ResponseStatusException(NOT_FOUND)); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/simple/EventuallyConsistentControllerTest.java ================================================ package com.gruelbox.transactionoutbox.spring.example.simple; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.web.server.LocalServerPort; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.web.client.RestClient; @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) class EventuallyConsistentControllerTest { @LocalServerPort private int port; private RestClient restClient; @Autowired private ExternalQueueService externalQueueService; @Autowired private JdbcTemplate jdbcTemplate; @BeforeEach void setUp() { this.restClient = RestClient.builder().baseUrl("http://localhost:" + port).build(); externalQueueService.clear(); } @Test void testCheckNormal() { var joe = new Customer(1L, "Joe", "Strummer"); var dave = new Customer(2L, "Dave", "Grohl"); var neil = new Customer(3L, "Neil", "Diamond"); var tupac = new Customer(4L, "Tupac", "Shakur"); var jeff = new Customer(5L, "Jeff", "Mills"); assertTrue( restClient .post() .uri("/customer") .body(joe) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/customer") .body(dave) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/customer") .body(neil) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/customer") .body(tupac) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/customer") .body(jeff) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); jdbcTemplate.execute( "UPDATE txno_outbox SET invocation='non-deserializable invocation' WHERE invocation LIKE '%" + neil.getLastName() + "%'"); await() .atMost(10, SECONDS) .pollDelay(1, SECONDS) .untilAsserted( () -> assertThat(externalQueueService.getSent()) .containsExactlyInAnyOrder(joe, dave, tupac, jeff)); } @Test void testCheckOrdered() { var joe = new Customer(1L, "Joe", "Strummer"); var dave = new Customer(2L, "Dave", "Grohl"); var neil = new Customer(3L, "Neil", "Diamond"); var tupac = new Customer(4L, "Tupac", "Shakur"); var jeff = new Customer(5L, "Jeff", "Mills"); assertTrue( restClient .post() .uri("/customer?ordered=true") .body(joe) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/customer?ordered=true") .body(dave) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/customer?ordered=true") .body(neil) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/customer?ordered=true") .body(tupac) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); assertTrue( restClient .post() .uri("/customer?ordered=true") .body(jeff) .retrieve() .toBodilessEntity() .getStatusCode() .is2xxSuccessful()); jdbcTemplate.execute( "UPDATE txno_outbox SET invocation='non-deserializable invocation' WHERE invocation LIKE '%" + neil.getLastName() + "%'"); await() .atMost(10, SECONDS) .pollDelay(1, SECONDS) .untilAsserted(() -> assertThat(externalQueueService.getSent()).containsExactly(joe, dave)); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/simple/ExternalQueueService.java ================================================ package com.gruelbox.transactionoutbox.spring.example.simple; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import lombok.Getter; import org.springframework.stereotype.Service; @Getter @Service class ExternalQueueService { private final Set attempted = new HashSet<>(); private final List sent = new CopyOnWriteArrayList<>(); void sendCustomerCreatedEvent(Customer customer) { if (attempted.add(customer.getId())) { throw new RuntimeException("Temporary failure, try again"); } sent.add(customer); } public void clear() { attempted.clear(); sent.clear(); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/simple/ExternalsConfiguration.java ================================================ package com.gruelbox.transactionoutbox.spring.example.simple; import com.gruelbox.transactionoutbox.spring.SpringInstantiator; import com.gruelbox.transactionoutbox.spring.SpringTransactionManager; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; @Configuration @Import({SpringInstantiator.class, SpringTransactionManager.class}) class ExternalsConfiguration {} ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/simple/TransactionOutboxBackgroundProcessor.java ================================================ package com.gruelbox.transactionoutbox.spring.example.simple; import com.gruelbox.transactionoutbox.TransactionOutbox; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; /** * Simple implementation of a background processor for {@link TransactionOutbox}. You don't need to * use this if you need different semantics, but this is a good start for most purposes. */ @Component @Slf4j @RequiredArgsConstructor(onConstructor_ = {@Autowired}) class TransactionOutboxBackgroundProcessor { private final TransactionOutbox outbox; @Scheduled(fixedRateString = "${outbox.repeatEvery}") void poll() { try { do { log.info("Flushing"); } while (outbox.flush()); } catch (Exception e) { log.error("Error flushing transaction outbox. Pausing", e); } } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/simple/TransactionOutboxProperties.java ================================================ package com.gruelbox.transactionoutbox.spring.example.simple; import java.time.Duration; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; @Configuration @ConfigurationProperties("outbox") @Data class TransactionOutboxProperties { private Duration repeatEvery; private boolean useJackson; private Duration attemptFrequency; private int blockAfterAttempts; } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/simple/TransactionOutboxSpringDemoApplication.java ================================================ package com.gruelbox.transactionoutbox.spring.example.simple; import com.fasterxml.jackson.databind.ObjectMapper; import com.gruelbox.transactionoutbox.DefaultPersistor; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.Persistor; import com.gruelbox.transactionoutbox.TransactionOutbox; import com.gruelbox.transactionoutbox.jackson.JacksonInvocationSerializer; import com.gruelbox.transactionoutbox.spring.SpringInstantiator; import com.gruelbox.transactionoutbox.spring.SpringTransactionManager; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Lazy; import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication @EnableScheduling public class TransactionOutboxSpringDemoApplication { public static void main(String[] args) { SpringApplication.run(TransactionOutboxSpringDemoApplication.class, args); } @Bean @Lazy Persistor persistor( TransactionOutboxProperties properties, @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") ObjectMapper objectMapper) { if (properties.isUseJackson()) { return DefaultPersistor.builder() .serializer(JacksonInvocationSerializer.builder().mapper(objectMapper).build()) .dialect(Dialect.H2) .build(); } else { return Persistor.forDialect(Dialect.H2); } } @Bean @Lazy TransactionOutbox transactionOutbox( SpringInstantiator instantiator, SpringTransactionManager transactionManager, TransactionOutboxProperties properties, Persistor persistor) { return TransactionOutbox.builder() .instantiator(instantiator) .transactionManager(transactionManager) .persistor(persistor) .attemptFrequency(properties.getAttemptFrequency()) .blockAfterAttempts(properties.getBlockAfterAttempts()) .build(); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/example/simple/Utils.java ================================================ package com.gruelbox.transactionoutbox.spring.example.simple; import com.gruelbox.transactionoutbox.ThrowingRunnable; import java.util.Arrays; import org.slf4j.Logger; import org.slf4j.LoggerFactory; class Utils { private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); @SuppressWarnings({"SameParameterValue", "WeakerAccess", "UnusedReturnValue"}) static boolean safelyRun(String gerund, ThrowingRunnable runnable) { try { runnable.run(); return true; } catch (Exception e) { LOGGER.error("Error when {}", gerund, e); return false; } } @SuppressWarnings("unused") static void safelyClose(AutoCloseable... closeables) { safelyClose(Arrays.asList(closeables)); } private static void safelyClose(Iterable closeables) { closeables.forEach( d -> { if (d == null) return; safelyRun("closing resource", d::close); }); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/it/MyRemoteService.java ================================================ package com.gruelbox.transactionoutbox.spring.it; import org.springframework.stereotype.Service; @Service public class MyRemoteService { public void execute() {} } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/it/SpringTransactionManagerIT.java ================================================ package com.gruelbox.transactionoutbox.spring.it; import com.gruelbox.transactionoutbox.AlreadyScheduledException; import com.gruelbox.transactionoutbox.TransactionOutbox; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.support.TransactionTemplate; @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) public class SpringTransactionManagerIT { @Autowired private TransactionOutbox outbox; @Autowired private PlatformTransactionManager transactionManager; @Test public void shouldThrowAlreadyScheduledException() { TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager); transactionTemplate.execute( status -> { outbox .with() .uniqueRequestId("my-unique-request") .schedule(MyRemoteService.class) .execute(); return null; }); transactionTemplate.execute( status -> { // Make sure we can't repeat the same work, and that we get expected exception Assertions.assertThrows( AlreadyScheduledException.class, () -> outbox .with() .uniqueRequestId("my-unique-request") .schedule(MyRemoteService.class) .execute()); return null; }); } } ================================================ FILE: transactionoutbox-spring/src/test/java/com/gruelbox/transactionoutbox/spring/it/TestApplication.java ================================================ package com.gruelbox.transactionoutbox.spring.it; import com.gruelbox.transactionoutbox.Dialect; import com.gruelbox.transactionoutbox.Persistor; import com.gruelbox.transactionoutbox.TransactionOutbox; import com.gruelbox.transactionoutbox.spring.SpringInstantiator; import com.gruelbox.transactionoutbox.spring.SpringTransactionManager; import java.time.Duration; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Lazy; @SpringBootApplication @Import({SpringInstantiator.class, SpringTransactionManager.class}) public class TestApplication { public static void main(String[] args) { SpringApplication.run(TestApplication.class, args); } @Bean @Lazy Persistor persistor() { return Persistor.forDialect(Dialect.H2); } @Bean @Lazy TransactionOutbox transactionOutbox( SpringInstantiator instantiator, SpringTransactionManager transactionManager, Persistor persistor) { return TransactionOutbox.builder() .instantiator(instantiator) .transactionManager(transactionManager) .persistor(persistor) .attemptFrequency(Duration.ofSeconds(1)) .blockAfterAttempts(3) .build(); } } ================================================ FILE: transactionoutbox-spring/src/test/resources/META-INF/persistence.xml ================================================ Hibernate EntityManager Demo com.gruelbox.transactionoutbox.acceptance.Dummy true ================================================ FILE: transactionoutbox-spring/src/test/resources/application.properties ================================================ server.port=8081 outbox.repeatEvery=PT1S outbox.attemptFrequency=PT0.5S outbox.blockAfterAttempts=100 outbox.useJackson=true ================================================ FILE: transactionoutbox-spring/src/test/resources/logback-test.xml ================================================ %d{HH:mm:ss.SSS} [%thread] %-5level %logger{5} - %msg%n ================================================ FILE: transactionoutbox-testing/pom.xml ================================================ transactionoutbox-parent com.gruelbox ${revision} 4.0.0 Transaction Outbox Testing jar transactionoutbox-testing A safe implementation of the transactional outbox pattern for Java (core library) com.gruelbox transactionoutbox-core ${project.version} compile org.projectlombok lombok ch.qos.logback logback-classic compile ch.qos.logback logback-core compile org.hamcrest hamcrest-core compile org.junit.jupiter junit-jupiter-engine compile org.junit.jupiter junit-jupiter-api compile org.junit.jupiter junit-jupiter-params compile org.mockito mockito-all compile com.zaxxer HikariCP compile io.opentelemetry opentelemetry-sdk-testing compile ================================================ FILE: transactionoutbox-testing/src/main/java/com/gruelbox/transactionoutbox/TestingMode.java ================================================ package com.gruelbox.transactionoutbox; public class TestingMode { public static void enable() { TransactionOutboxImpl.FORCE_SERIALIZE_AND_DESERIALIZE_BEFORE_USE.set(true); } public static void disable() { TransactionOutboxImpl.FORCE_SERIALIZE_AND_DESERIALIZE_BEFORE_USE.set(false); } } ================================================ FILE: transactionoutbox-testing/src/main/java/com/gruelbox/transactionoutbox/testing/AbstractAcceptanceTest.java ================================================ package com.gruelbox.transactionoutbox.testing; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; import static org.junit.jupiter.api.Assertions.*; import com.gruelbox.transactionoutbox.*; import com.gruelbox.transactionoutbox.spi.Utils; import com.zaxxer.hikari.HikariDataSource; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.TraceFlags; import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; import io.opentelemetry.sdk.trace.data.SpanData; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.SQLException; import java.time.Clock; import java.time.Duration; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.IntStream; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @Slf4j public abstract class AbstractAcceptanceTest extends BaseTest { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractAcceptanceTest.class); private ExecutorService unreliablePool; private ExecutorService singleThreadPool; private static final Random random = new Random(); @BeforeEach void beforeEachBase() { unreliablePool = new ThreadPoolExecutor(2, 2, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(16)); singleThreadPool = Executors.newSingleThreadExecutor(); } @AfterEach void afterEachBase() throws InterruptedException { unreliablePool.shutdown(); singleThreadPool.shutdown(); assertTrue(unreliablePool.awaitTermination(30, SECONDS)); assertTrue(singleThreadPool.awaitTermination(30, SECONDS)); } @Test final void testMDCPassedToTask() throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); var transactionManager = txManager(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .instantiator( Instantiator.using( clazz -> (InterfaceProcessor) (foo, bar) -> { log.info("Processing ({}, {})", foo, bar); assertEquals("Foo", MDC.get("SESSION-KEY")); })) .listener(new LatchListener(latch)) .persistor(StubPersistor.builder().build()) .build(); MDC.put("SESSION-KEY", "Foo"); try { transactionManager.inTransaction( () -> outbox.schedule(InterfaceProcessor.class).process(3, "Whee")); } finally { MDC.clear(); } assertTrue(latch.await(2, TimeUnit.SECONDS)); } @Test final void sequencing() throws Exception { int countPerTopic = 20; int topicCount = 5; AtomicInteger insertIndex = new AtomicInteger(); CountDownLatch latch = new CountDownLatch(countPerTopic * topicCount); ThreadLocalContextTransactionManager transactionManager = (ThreadLocalContextTransactionManager) txManager(); transactionManager.inTransaction( tx -> { //noinspection resource try (var stmt = tx.connection().createStatement()) { stmt.execute("DROP TABLE TEST_TABLE"); } catch (SQLException e) { // ignore } }); transactionManager.inTransaction( tx -> { //noinspection resource try (var stmt = tx.connection().createStatement()) { stmt.execute(createTestTable()); } catch (SQLException e) { throw new RuntimeException(e); } }); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .submitter(Submitter.withExecutor(unreliablePool)) .attemptFrequency(Duration.ofMillis(500)) .instantiator( new RandomFailingInstantiator( (foo, bar) -> { transactionManager.requireTransaction( tx -> { //noinspection resource try (var stmt = tx.connection() .prepareStatement( "INSERT INTO TEST_TABLE (topic, ix, foo) VALUES(?, ?, ?)")) { stmt.setString(1, bar); stmt.setInt(2, insertIndex.incrementAndGet()); stmt.setInt(3, foo); stmt.executeUpdate(); } catch (SQLException e) { throw new RuntimeException(e); } }); })) .persistor(persistor()) .listener(new LatchListener(latch)) .initializeImmediately(false) .flushBatchSize(4) .build(); outbox.initialize(); clearOutbox(); withRunningFlusher( outbox, () -> { transactionManager.inTransaction( () -> { for (int i = 1; i <= countPerTopic; i++) { for (int j = 1; j <= topicCount; j++) { outbox .with() .ordered("topic" + j) .schedule(InterfaceProcessor.class) .process(i, "topic" + j); } } }); assertTrue(latch.await(30, SECONDS)); }); var output = new HashMap>(); transactionManager.inTransaction( tx -> { //noinspection resource try (var stmt = tx.connection().createStatement(); var rs = stmt.executeQuery("SELECT topic, foo FROM TEST_TABLE ORDER BY ix")) { while (rs.next()) { ArrayList values = output.computeIfAbsent(rs.getString(1), k -> new ArrayList<>()); values.add(rs.getInt(2)); } } catch (SQLException e) { throw new RuntimeException(e); } }); var indexes = IntStream.range(1, countPerTopic + 1).boxed().collect(toList()); var expected = IntStream.range(1, topicCount + 1) .mapToObj(i -> "topic" + i) .collect(toMap(it -> it, it -> indexes)); assertEquals(expected, output); } /** * Uses a simple direct transaction manager and connection manager and attempts to fire an * interface using a custom instantiator. */ @Test final void simpleConnectionProviderCustomInstantiatorInterfaceClass() throws InterruptedException { TransactionManager transactionManager = txManager(); CountDownLatch latch = new CountDownLatch(1); CountDownLatch chainedLatch = new CountDownLatch(1); AtomicBoolean gotScheduled = new AtomicBoolean(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .instantiator( Instantiator.using( clazz -> (InterfaceProcessor) (foo, bar) -> LOGGER.info("Processing ({}, {})", foo, bar))) .submitter(Submitter.withExecutor(unreliablePool)) .listener( new LatchListener(latch) .andThen( new TransactionOutboxListener() { @Override public void scheduled(TransactionOutboxEntry entry) { log.info("Got scheduled event"); gotScheduled.set(true); } @Override public void success(TransactionOutboxEntry entry) { chainedLatch.countDown(); } })) .persistor(persistor()) .initializeImmediately(false) .build(); outbox.initialize(); clearOutbox(); transactionManager.inTransaction( () -> { outbox.schedule(InterfaceProcessor.class).process(3, "Whee"); try { // Should not be fired until after commit assertFalse(latch.await(2, SECONDS)); } catch (InterruptedException e) { fail("Interrupted"); } }); // Should be fired after commit assertTrue(chainedLatch.await(2, SECONDS)); assertTrue(latch.await(1, SECONDS)); assertTrue(gotScheduled.get()); } @Test final void noAutomaticInitialization() { TransactionManager transactionManager = txManager(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .instantiator( Instantiator.using( clazz -> (InterfaceProcessor) (foo, bar) -> LOGGER.info("Processing ({}, {})", foo, bar))) .submitter(Submitter.withDefaultExecutor()) .persistor(Persistor.forDialect(connectionDetails().dialect())) .initializeImmediately(false) .build(); Persistor.forDialect(connectionDetails().dialect()).migrate(txManager()); clearOutbox(); Assertions.assertThrows( IllegalStateException.class, () -> transactionManager.inTransaction( () -> outbox.schedule(InterfaceProcessor.class).process(3, "Whee"))); } @Test void duplicateRequests() { TransactionManager transactionManager = txManager(); List ids = new ArrayList<>(); AtomicReference clockProvider = new AtomicReference<>(Clock.systemDefaultZone()); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { ids.add((String) entry.getInvocation().getArgs()[0]); } }) .submitter(Submitter.withExecutor(Runnable::run)) .persistor(Persistor.forDialect(connectionDetails().dialect())) .retentionThreshold(Duration.ofDays(2)) .clockProvider(clockProvider::get) .build(); clearOutbox(); // Schedule some work transactionManager.inTransaction( () -> outbox .with() .uniqueRequestId("context-clientkey1") .schedule(ClassProcessor.class) .process("1")); // Make sure we can schedule more work with a different client key transactionManager.inTransaction( () -> outbox .with() .uniqueRequestId("context-clientkey2") .schedule(ClassProcessor.class) .process("2")); // Make sure we can't repeat the same work transactionManager.inTransaction( () -> Assertions.assertThrows( AlreadyScheduledException.class, () -> outbox .with() .uniqueRequestId("context-clientkey1") .schedule(ClassProcessor.class) .process("3"))); // Run the clock forward to just under the retention threshold clockProvider.set( Clock.fixed( clockProvider.get().instant().plus(Duration.ofDays(2)).minusSeconds(60), clockProvider.get().getZone())); outbox.flush(); // Make sure we can schedule more work with a different client key transactionManager.inTransaction( () -> outbox .with() .uniqueRequestId("context-clientkey4") .schedule(ClassProcessor.class) .process("4")); // Make sure we still can't repeat the same work transactionManager.inTransaction( () -> Assertions.assertThrows( AlreadyScheduledException.class, () -> outbox .with() .uniqueRequestId("context-clientkey1") .schedule(ClassProcessor.class) .process("5"))); // Run the clock over the threshold clockProvider.set( Clock.fixed(clockProvider.get().instant().plusSeconds(120), clockProvider.get().getZone())); outbox.flush(); // We should now be able to add the work transactionManager.inTransaction( () -> outbox .with() .uniqueRequestId("context-clientkey1") .schedule(ClassProcessor.class) .process("6")); assertThat(ids, containsInAnyOrder("1", "2", "4", "6")); } /** * Uses a simple data source transaction manager and attempts to fire a concrete class via * reflection. */ @Test final void dataSourceConnectionProviderReflectionInstantiatorConcreteClass() throws InterruptedException { try (HikariDataSource ds = dataSource) { CountDownLatch latch = new CountDownLatch(1); TransactionManager transactionManager = TransactionManager.fromDataSource(ds); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(connectionDetails().dialect())) .listener(new LatchListener(latch)) .build(); clearOutbox(); ClassProcessor.PROCESSED.clear(); String myId = UUID.randomUUID().toString(); transactionManager.inTransaction(() -> outbox.schedule(ClassProcessor.class).process(myId)); assertTrue(latch.await(2, SECONDS)); assertEquals(List.of(myId), ClassProcessor.PROCESSED); } } /** * Implements a custom transaction manager. Any required changes to this test are a sign that we * need to bump the major revision. */ @Test final void customTransactionManager() throws ClassNotFoundException, SQLException, InterruptedException { Class.forName(connectionDetails().driverClassName()); try (Connection connection = DriverManager.getConnection( connectionDetails().url(), connectionDetails().user(), connectionDetails().password())) { connection.setAutoCommit(false); connection.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); ArrayList postCommitHooks = new ArrayList<>(); ArrayList preparedStatements = new ArrayList<>(); CountDownLatch latch = new CountDownLatch(1); Transaction transaction = new Transaction() { @Override public Connection connection() { return connection; } @Override @SneakyThrows public PreparedStatement prepareBatchStatement(String sql) { var stmt = connection.prepareStatement(sql); preparedStatements.add(stmt); return stmt; } @Override public void addPostCommitHook(Runnable runnable) { postCommitHooks.add(runnable); } }; TransactionManager transactionManager = new ThreadLocalContextTransactionManager() { @Override public T inTransactionReturnsThrows( ThrowingTransactionalSupplier work) throws E { return work.doWork(transaction); } @Override public T requireTransactionReturns( ThrowingTransactionalSupplier work) throws E, NoTransactionActiveException { return work.doWork(transaction); } }; TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .listener(new LatchListener(latch)) .persistor(Persistor.forDialect(connectionDetails().dialect())) .build(); clearOutbox(); ClassProcessor.PROCESSED.clear(); String myId = UUID.randomUUID().toString(); try { outbox.schedule(ClassProcessor.class).process(myId); preparedStatements.forEach( it -> { try { it.executeBatch(); it.close(); } catch (SQLException e) { throw new RuntimeException(e); } }); connection.commit(); } catch (Exception e) { connection.rollback(); throw e; } postCommitHooks.forEach(Runnable::run); assertTrue(latch.await(2, SECONDS)); assertEquals(List.of(myId), ClassProcessor.PROCESSED); } } /** * Runs a piece of work which will fail several times before working successfully. Ensures that * the work runs eventually. */ @Test final void retryBehaviour() throws Exception { TransactionManager transactionManager = txManager(); CountDownLatch latch = new CountDownLatch(1); AtomicInteger attempts = new AtomicInteger(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(connectionDetails().dialect())) .instantiator(new FailingInstantiator(attempts)) .submitter(Submitter.withExecutor(singleThreadPool)) .attemptFrequency(Duration.ofMillis(500)) .listener(new LatchListener(latch)) .build(); clearOutbox(); withRunningFlusher( outbox, () -> { transactionManager.inTransaction( () -> outbox.schedule(InterfaceProcessor.class).process(3, "Whee")); assertTrue(latch.await(15, SECONDS)); }, singleThreadPool); } @Test final void flushOnlyASpecifiedTopic() throws Exception { TransactionManager transactionManager = txManager(); CountDownLatch successLatch = new CountDownLatch(1); var processedEntryListener = new ProcessedEntryListener(successLatch); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(connectionDetails().dialect())) .instantiator( Instantiator.using( clazz -> (InterfaceProcessor) (foo, bar) -> LOGGER.info( "Entered the method to process successfully. Processing ({}, {})", foo, bar))) .submitter(Submitter.withExecutor(singleThreadPool)) .attemptFrequency(Duration.ofMillis(500)) .listener(processedEntryListener) .build(); clearOutbox(); var selectedTopic = "SELECTED_TOPIC"; transactionManager.inTransaction( () -> { outbox .with() .ordered(selectedTopic) .schedule(InterfaceProcessor.class) .process(1, "Whoo"); outbox .with() .ordered("IGNORED_TOPIC") .schedule(InterfaceProcessor.class) .process(2, "Wheeeee"); }); assertFalse( successLatch.await(5, SECONDS), "At this point, nothing should have been picked up for processing"); outbox.flushTopics(singleThreadPool, selectedTopic); assertTrue(successLatch.await(5, SECONDS), "Should have successfully processed something"); var successes = processedEntryListener.getSuccessfulEntries(); var failures = processedEntryListener.getFailingEntries(); // then we only expect the selected topic we're flushing to have had eventually succeeded // as the other work would not have been picked up for a retry assertEquals(1, successes.stream().map(TransactionOutboxEntry::getTopic).distinct().count()); assertEquals(selectedTopic, successes.get(0).getTopic()); // no failures expected assertEquals(0, failures.size()); } @Test final void onSchedulingFailure_BubbleExceptionsUp() throws Exception { Assumptions.assumeTrue( Dialect.MY_SQL_8.equals(connectionDetails().dialect()) || Dialect.MY_SQL_5.equals(connectionDetails().dialect())); TransactionManager transactionManager = txManager(); CountDownLatch latch = new CountDownLatch(1); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .instantiator( Instantiator.using( clazz -> (InterfaceProcessor) (foo, bar) -> LOGGER.info( "Entered the method to process successfully. Processing ({}, {})", foo, bar))) .persistor(Persistor.forDialect(connectionDetails().dialect())) .submitter(Submitter.withExecutor(unreliablePool)) .attemptFrequency(Duration.ofMillis(500)) .listener(new LatchListener(latch)) .build(); clearOutbox(); withRunningFlusher( outbox, () -> assertThrows( Exception.class, () -> transactionManager.inTransaction( () -> outbox .with() .uniqueRequestId("some_unique_id") .schedule(InterfaceProcessor.class) .process(1, "This invocation is too long".repeat(650000))))); } @Test final void lastAttemptTime_updatesEveryTime() throws Exception { TransactionManager transactionManager = txManager(); CountDownLatch successLatch = new CountDownLatch(1); CountDownLatch blockLatch = new CountDownLatch(1); AtomicInteger attempts = new AtomicInteger(); var orderedEntryListener = new OrderedEntryListener(successLatch, blockLatch); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(connectionDetails().dialect())) .instantiator(new FailingInstantiator(attempts)) .submitter(Submitter.withExecutor(singleThreadPool)) .attemptFrequency(Duration.ofMillis(500)) .listener(orderedEntryListener) .blockAfterAttempts(2) .build(); clearOutbox(); withRunningFlusher( outbox, () -> { transactionManager.inTransaction( () -> outbox.schedule(InterfaceProcessor.class).process(3, "Whee")); assertTrue(blockLatch.await(20, SECONDS), "Entry was not blocked"); assertTrue( (Boolean) transactionManager.inTransactionReturns( tx -> outbox.unblock(orderedEntryListener.getBlocked().getId()))); assertTrue(successLatch.await(20, SECONDS), "Timeout waiting for success"); var events = orderedEntryListener.getEvents(); log.info("The entry life cycle is: {}", events); // then we are only dealing in terms of a single outbox entry. assertEquals(1, events.stream().map(TransactionOutboxEntry::getId).distinct().count()); // the first, scheduled entry has no lastAttemptTime set assertNull(events.get(0).getLastAttemptTime()); // all subsequent entries (2 x failures (second of which 'blocks'), 1x success updates // against db) have a distinct lastAttemptTime set on them. assertEquals( 3, events.stream() .skip(1) .map(TransactionOutboxEntry::getLastAttemptTime) .distinct() .count()); }, singleThreadPool); } /** * Runs a piece of work which will fail enough times to enter a blocked state but will then pass * when re-tried after it is unblocked. */ @Test final void blockAndThenUnblockForRetry() throws Exception { TransactionManager transactionManager = txManager(); CountDownLatch successLatch = new CountDownLatch(1); CountDownLatch blockLatch = new CountDownLatch(1); LatchListener latchListener = new LatchListener(successLatch, blockLatch); AtomicInteger attempts = new AtomicInteger(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(connectionDetails().dialect())) .instantiator(new FailingInstantiator(attempts)) .submitter(Submitter.withExecutor(singleThreadPool)) .attemptFrequency(Duration.ofMillis(500)) .listener(latchListener) .blockAfterAttempts(2) .build(); clearOutbox(); withRunningFlusher( outbox, () -> { transactionManager.inTransaction( () -> outbox.schedule(InterfaceProcessor.class).process(3, "Whee")); assertTrue(blockLatch.await(5, SECONDS)); assertTrue( (Boolean) transactionManager.inTransactionReturns( tx -> outbox.unblock(latchListener.getBlocked().getId()))); assertTrue(successLatch.await(5, SECONDS)); }, singleThreadPool); } /** Hammers high-volume, frequently failing tasks to ensure that they all get run. */ @Test final void highVolumeUnreliable() throws Exception { int count = 10; TransactionManager transactionManager = txManager(); CountDownLatch latch = new CountDownLatch(count * 10); ConcurrentHashMap results = new ConcurrentHashMap<>(); ConcurrentHashMap duplicates = new ConcurrentHashMap<>(); TransactionOutbox outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(Persistor.forDialect(connectionDetails().dialect())) .instantiator(new RandomFailingInstantiator()) .submitter(Submitter.withExecutor(unreliablePool)) .attemptFrequency(Duration.ofMillis(500)) .flushBatchSize(1000) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { Integer i = (Integer) entry.getInvocation().getArgs()[0]; if (results.putIfAbsent(i, i) != null) { duplicates.put(i, i); } latch.countDown(); } }) .build(); withRunningFlusher( outbox, () -> { IntStream.range(0, count) .parallel() .forEach( i -> transactionManager.inTransaction( () -> { for (int j = 0; j < 10; j++) { outbox.schedule(InterfaceProcessor.class).process(i * 10 + j, "Whee"); } })); assertTrue(latch.await(30, SECONDS), "Latch not opened in time"); }); assertThat( "Should never get duplicates running to full completion", duplicates.keySet(), empty()); assertThat( "Only got: " + results.keySet(), results.keySet(), containsInAnyOrder(IntStream.range(0, count * 10).boxed().toArray())); } protected String createTestTable() { return "CREATE TABLE TEST_TABLE (topic VARCHAR(50), ix INTEGER, foo INTEGER, PRIMARY KEY (topic, ix))"; } private static class FailingInstantiator implements Instantiator { private final AtomicInteger attempts; FailingInstantiator(AtomicInteger attempts) { this.attempts = attempts; } @Override public String getName(Class clazz) { return "BEEF"; } @Override public Object getInstance(String name) { if (!"BEEF".equals(name)) { throw new UnsupportedOperationException(); } return (InterfaceProcessor) (foo, bar) -> { LOGGER.info("Processing ({}, {})", foo, bar); if (attempts.incrementAndGet() < 3) { throw new RuntimeException("Temporary failure"); } LOGGER.info("Processed ({}, {})", foo, bar); }; } } private static class RandomFailingInstantiator implements Instantiator { private final InterfaceProcessor interfaceProcessor; RandomFailingInstantiator() { this.interfaceProcessor = (foo, bar) -> {}; } RandomFailingInstantiator(InterfaceProcessor interfaceProcessor) { this.interfaceProcessor = interfaceProcessor; } @Override public String getName(Class clazz) { return clazz.getName(); } @Override public Object getInstance(String name) { if (InterfaceProcessor.class.getName().equals(name)) { return (InterfaceProcessor) (foo, bar) -> { if (random.nextInt(10) == 5) { throw new RuntimeException("Temporary failure of InterfaceProcessor"); } interfaceProcessor.process(foo, bar); }; } else { throw new UnsupportedOperationException(); } } } @Test void runWithParentOtelSpan() throws Exception { OpenTelemetryExtension otelTesting = OpenTelemetryExtension.create(); OpenTelemetry otel = otelTesting.getOpenTelemetry(); otelTesting.beforeAll(null); try { otelTesting.beforeEach(null); var latch = new CountDownLatch(1); AtomicReference remotedSpan = new AtomicReference<>(); var txManager = txManager(); var outbox = TransactionOutbox.builder() .transactionManager(txManager) .persistor(Persistor.forDialect(connectionDetails().dialect())) .instantiator( Instantiator.using( clazz -> (InterfaceProcessor) (foo, bar) -> { remotedSpan.set(Span.current().getSpanContext()); })) .attemptFrequency(Duration.ofMillis(500)) .listener(new OtelListener().andThen(new LatchListener(latch))) .blockAfterAttempts(2) .build(); // Start a parent span, which should be propagated to the instantiator above Span parentSpan = otel.getTracer("parent-tracer").spanBuilder("parent-span").startSpan(); String parentTraceId = null; try (Scope scope = parentSpan.makeCurrent()) { parentTraceId = Span.current().getSpanContext().getTraceId(); txManager.inTransaction(() -> outbox.schedule(InterfaceProcessor.class).process(1, "1")); } finally { parentSpan.end(); } // Wait for the job to complete assertTrue(latch.await(10, TimeUnit.SECONDS)); SpanData remotedSpanData = null; for (int i = 0; i < 5; i++) { remotedSpanData = otelTesting.getSpans().stream() .filter(it -> it.getSpanId().equals(remotedSpan.get().getSpanId())) .findFirst() .orElse(null); if (remotedSpanData == null) { if (i == 4) { throw new RuntimeException("No matching span"); } else { Thread.sleep(500); } } } // Check they ran with linked traces and the correct class/method/args assertTrue( remotedSpanData.getLinks().stream() .findFirst() .orElseThrow(() -> new RuntimeException("No linked trace")) .getSpanContext() .getTraceId() .equals(parentTraceId)); assertTrue( remotedSpanData .getName() .equals("com.gruelbox.transactionoutbox.testing.InterfaceProcessor.process")); assertTrue(remotedSpanData.getAttributes().get(AttributeKey.stringKey("arg0")).equals("1")); assertTrue( remotedSpanData.getAttributes().get(AttributeKey.stringKey("arg1")).equals("\"1\"")); } finally { otelTesting.afterAll(null); } } @Test void runWithoutParentOtelSpan() throws Exception { OpenTelemetryExtension otelTesting = OpenTelemetryExtension.create(); OpenTelemetry otel = otelTesting.getOpenTelemetry(); otelTesting.beforeAll(null); try { otelTesting.beforeEach(null); var latch = new CountDownLatch(1); AtomicReference remotedSpan = new AtomicReference<>(); var txManager = txManager(); var outbox = TransactionOutbox.builder() .transactionManager(txManager) .persistor(Persistor.forDialect(connectionDetails().dialect())) .instantiator( Instantiator.using( clazz -> (InterfaceProcessor) (foo, bar) -> remotedSpan.set(Span.current().getSpanContext()))) .attemptFrequency(Duration.ofMillis(500)) .listener(new OtelListener().andThen(new LatchListener(latch))) .blockAfterAttempts(2) .build(); // Run with no parent span txManager.inTransaction(() -> outbox.schedule(InterfaceProcessor.class).process(1, "1")); // Wait for the job to complete assertTrue(latch.await(10, TimeUnit.SECONDS)); SpanData remotedSpanData = null; for (int i = 0; i < 5; i++) { remotedSpanData = otelTesting.getSpans().stream() .filter(it -> it.getSpanId().equals(remotedSpan.get().getSpanId())) .findFirst() .orElse(null); if (remotedSpanData == null) { if (i == 4) { throw new RuntimeException("No matching span"); } else { Thread.sleep(500); } } } // Check they ran with linked traces and the correct class/method/args assertFalse(remotedSpanData.getLinks().stream().findFirst().isPresent()); assertTrue( remotedSpanData .getName() .equals("com.gruelbox.transactionoutbox.testing.InterfaceProcessor.process")); assertTrue(remotedSpanData.getAttributes().get(AttributeKey.stringKey("arg0")).equals("1")); assertTrue( remotedSpanData.getAttributes().get(AttributeKey.stringKey("arg1")).equals("\"1\"")); } finally { otelTesting.afterAll(null); } } /** Example {@link TransactionOutboxListener} to propagate traces */ static class OtelListener implements TransactionOutboxListener { /** Serialises the current context into {@link Invocation#getSession()}. */ @Override public Map extractSession() { var result = new HashMap(); SpanContext spanContext = Span.current().getSpanContext(); if (!spanContext.isValid()) { return null; } result.put("traceId", spanContext.getTraceId()); result.put("spanId", spanContext.getSpanId()); log.info("Extracted: {}", result); return result; } /** * Deserialises {@link Invocation#getSession()} and sets it as the current context so that any * new span started by the method we invoke will treat it as the parent span */ @Override public void wrapInvocationAndInit(Invocator invocator) { Invocation inv = invocator.getInvocation(); var spanBuilder = GlobalOpenTelemetry.get() .getTracer("transaction-outbox") .spanBuilder(String.format("%s.%s", inv.getClassName(), inv.getMethodName())) .setNoParent(); for (var i = 0; i < inv.getArgs().length; i++) { spanBuilder.setAttribute("arg" + i, Utils.stringify(inv.getArgs()[i])); } if (inv.getSession() != null) { var traceId = inv.getSession().get("traceId"); var spanId = inv.getSession().get("spanId"); if (traceId != null && spanId != null) { spanBuilder.addLink( SpanContext.createFromRemoteParent( traceId, spanId, TraceFlags.getSampled(), TraceState.getDefault())); } } var span = spanBuilder.startSpan(); try (Scope scope = span.makeCurrent()) { invocator.runUnchecked(); } finally { span.end(); } } } } ================================================ FILE: transactionoutbox-testing/src/main/java/com/gruelbox/transactionoutbox/testing/AbstractPersistorTest.java ================================================ package com.gruelbox.transactionoutbox.testing; import static java.time.temporal.ChronoUnit.MILLIS; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; import static org.junit.jupiter.api.Assertions.*; import com.gruelbox.transactionoutbox.*; import java.io.IOException; import java.math.BigDecimal; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.concurrent.*; import lombok.extern.slf4j.Slf4j; import org.hamcrest.Description; import org.hamcrest.TypeSafeMatcher; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @Slf4j public abstract class AbstractPersistorTest { private final Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); protected abstract Dialect dialect(); protected abstract Persistor persistor(); protected abstract TransactionManager txManager(); protected void validateState() {} @BeforeEach public void beforeEach() throws Exception { Boolean connected = txManager().inTransactionReturnsThrows(tx -> persistor().checkConnection(tx)); assertTrue(connected); persistor().migrate(txManager()); log.info("Validating state"); validateState(); log.info("Clearing old records"); txManager().inTransactionThrows(persistor()::clear); log.info("Cleared"); } @Test public void testInsertAndSelect() throws Exception { TransactionOutboxEntry entry = createEntry("FOO", now, false); txManager().inTransactionThrows(tx -> persistor().save(tx, entry)); Thread.sleep(1100); txManager() .inTransactionThrows( tx -> assertThat(persistor().selectBatch(tx, 100, now.plusMillis(1)), contains(entry))); } @Test public void testInsertWithUniqueRequestIdFailureBubblesExceptionUp() { var invalidEntry = createEntry("FOO", now, false).toBuilder() .uniqueRequestId("INTENTIONALLY_TOO_LONG_TO_CAUSE_BLOW_UP".repeat(10)) .build(); assertThrows( RuntimeException.class, () -> txManager().inTransactionThrows(tx -> persistor().save(tx, invalidEntry))); } @Test public void testInsertDuplicate() throws Exception { TransactionOutboxEntry entry1 = createEntry("FOO1", now, false, "context-clientkey1"); txManager().inTransactionThrows(tx -> persistor().save(tx, entry1)); Thread.sleep(1100); txManager() .inTransactionThrows( tx -> assertThat(persistor().selectBatch(tx, 100, now.plusMillis(1)), contains(entry1))); TransactionOutboxEntry entry2 = createEntry("FOO2", now, false, "context-clientkey2"); txManager().inTransactionThrows(tx -> persistor().save(tx, entry2)); Thread.sleep(1100); txManager() .inTransactionThrows( tx -> assertThat( persistor().selectBatch(tx, 100, now.plusMillis(1)), containsInAnyOrder(entry1, entry2))); TransactionOutboxEntry entry3 = createEntry("FOO3", now, false, "context-clientkey1"); Assertions.assertThrows( AlreadyScheduledException.class, () -> txManager().inTransactionThrows(tx -> persistor().save(tx, entry3))); txManager() .inTransactionThrows( tx -> assertThat( persistor().selectBatch(tx, 100, now.plusMillis(1)), containsInAnyOrder(entry1, entry2))); } @Test public void testBatchLimitUnderThreshold() throws Exception { txManager() .inTransactionThrows( tx -> { persistor().save(tx, createEntry("FOO1", now, false)); persistor().save(tx, createEntry("FOO2", now, false)); persistor().save(tx, createEntry("FOO3", now, false)); }); txManager() .inTransactionThrows( tx -> assertThat(persistor().selectBatch(tx, 2, now.plusMillis(1)), hasSize(2))); } @Test public void testBatchLimitMatchingThreshold() throws Exception { txManager() .inTransactionThrows( tx -> { persistor().save(tx, createEntry("FOO1", now, false)); persistor().save(tx, createEntry("FOO2", now, false)); persistor().save(tx, createEntry("FOO3", now, false)); }); txManager() .inTransactionThrows( tx -> assertThat(persistor().selectBatch(tx, 3, now.plusMillis(1)), hasSize(3))); } @Test public void testBatchLimitOverThreshold() throws Exception { txManager() .inTransactionThrows( tx -> { persistor().save(tx, createEntry("FOO1", now, false)); persistor().save(tx, createEntry("FOO2", now, false)); persistor().save(tx, createEntry("FOO3", now, false)); }); txManager() .inTransactionThrows( tx -> assertThat(persistor().selectBatch(tx, 4, now.plusMillis(1)), hasSize(3))); } @Test public void testBatchHorizon() throws Exception { txManager() .inTransactionThrows( tx -> { persistor().save(tx, createEntry("FOO1", now, false)); persistor().save(tx, createEntry("FOO2", now, false)); persistor().save(tx, createEntry("FOO3", now.plusMillis(2), false)); }); txManager() .inTransactionThrows( tx -> assertThat(persistor().selectBatch(tx, 3, now.plusMillis(1)), hasSize(2))); } @Test public void testBlockedEntriesExcluded() throws Exception { txManager() .inTransactionThrows( tx -> { persistor().save(tx, createEntry("FOO1", now, false)); persistor().save(tx, createEntry("FOO2", now, false)); persistor().save(tx, createEntry("FOO3", now, true)); }); txManager() .inTransactionThrows( tx -> assertThat(persistor().selectBatch(tx, 3, now.plusMillis(1)), hasSize(2))); } @Test public void testUnparseableEntriesExcluded() throws Exception { txManager() .inTransactionThrows( tx -> { persistor().save(tx, createEntry("FOO1", now, false)); persistor().save(tx, createEntry("FOO2", now, createUnparseableInvocation())); persistor().save(tx, createEntry("FOO3", now, false)); }); txManager() .inTransactionThrows( tx -> assertThat(persistor().selectBatch(tx, 3, now.plusMillis(1)), hasSize(3))); } static class TransactionOutboxEntryMatcher extends TypeSafeMatcher { private final TransactionOutboxEntry entry; TransactionOutboxEntryMatcher(TransactionOutboxEntry entry) { this.entry = entry; } @Override protected boolean matchesSafely(TransactionOutboxEntry other) { return entry.getId().equals(other.getId()) && entry.getInvocation().equals(other.getInvocation()) && entry.getNextAttemptTime().equals(other.getNextAttemptTime()) && entry.getAttempts() == other.getAttempts() && entry.getVersion() == other.getVersion() && entry.isBlocked() == other.isBlocked() && entry.isProcessed() == other.isProcessed(); } @Override public void describeTo(Description description) { description .appendText("Should match on all fields outside of lastAttemptTime :") .appendText(entry.toString()); } } TransactionOutboxEntryMatcher matches(TransactionOutboxEntry e) { return new TransactionOutboxEntryMatcher(e); } @Test public void testUpdate() throws Exception { var entry = createEntry("FOO1", now, false); txManager().inTransactionThrows(tx -> persistor().save(tx, entry)); entry.setAttempts(1); txManager().inTransaction(tx -> assertDoesNotThrow(() -> persistor().update(tx, entry))); var updatedEntry1 = txManager() .inTransactionReturnsThrows(tx -> persistor().selectBatch(tx, 1, now.plusMillis(1))); assertThat(updatedEntry1.size(), equalTo(1)); assertThat(updatedEntry1.get(0), matches(entry)); assertThat(updatedEntry1.get(0).getLastAttemptTime(), nullValue()); entry.setAttempts(2); entry.setLastAttemptTime(now); txManager().inTransaction(tx -> assertDoesNotThrow(() -> persistor().update(tx, entry))); var updatedEntry2 = txManager() .inTransactionReturnsThrows(tx -> persistor().selectBatch(tx, 1, now.plusMillis(1))); assertThat(updatedEntry2.size(), equalTo(1)); assertThat(updatedEntry2.get(0), matches(entry)); assertThat(updatedEntry2.get(0).getLastAttemptTime(), notNullValue()); } @Test public void testUpdateOptimisticLockFailure() throws Exception { TransactionOutboxEntry entry = createEntry("FOO1", now, false); txManager().inTransactionThrows(tx -> persistor().save(tx, entry)); TransactionOutboxEntry original = entry.toBuilder().build(); entry.setAttempts(1); txManager().inTransaction(tx -> assertDoesNotThrow(() -> persistor().update(tx, entry))); original.setAttempts(2); txManager() .inTransaction( tx -> assertThrows( OptimisticLockException.class, () -> persistor().update(tx, original))); } @Test public void testDelete() throws Exception { TransactionOutboxEntry entry = createEntry("FOO1", now, false); txManager().inTransactionThrows(tx -> persistor().save(tx, entry)); txManager().inTransaction(tx -> assertDoesNotThrow(() -> persistor().delete(tx, entry))); } @Test public void testDeleteOptimisticLockFailure() throws Exception { TransactionOutboxEntry entry = createEntry("FOO1", now, false); txManager().inTransactionThrows(tx -> persistor().save(tx, entry)); txManager().inTransaction(tx -> assertDoesNotThrow(() -> persistor().delete(tx, entry))); txManager() .inTransaction( tx -> assertThrows(OptimisticLockException.class, () -> persistor().delete(tx, entry))); } @Test public void testLock() throws Exception { TransactionOutboxEntry entry = createEntry("FOO1", now, false); txManager().inTransactionThrows(tx -> persistor().save(tx, entry)); entry.setAttempts(1); txManager().inTransaction(tx -> assertDoesNotThrow(() -> persistor().update(tx, entry))); txManager().inTransactionThrows(tx -> assertThat(persistor().lock(tx, entry), equalTo(true))); } @Test public void testLockOptimisticLockFailure() throws Exception { TransactionOutboxEntry entry = createEntry("FOO1", now, false); txManager().inTransactionThrows(tx -> persistor().save(tx, entry)); TransactionOutboxEntry original = entry.toBuilder().build(); entry.setAttempts(1); txManager().inTransaction(tx -> assertDoesNotThrow(() -> persistor().update(tx, entry))); txManager() .inTransactionThrows(tx -> assertThat(persistor().lock(tx, original), equalTo(false))); } @Test public void testSkipLocked() throws Exception { var entry1 = createEntry("FOO1", now.minusSeconds(1), false); var entry2 = createEntry("FOO2", now.minusSeconds(1), false); var entry3 = createEntry("FOO3", now.minusSeconds(1), false); var entry4 = createEntry("FOO4", now.minusSeconds(1), false); txManager() .inTransactionThrows( tx -> { persistor().save(tx, entry1); persistor().save(tx, entry2); persistor().save(tx, entry3); persistor().save(tx, entry4); }); var gotLockLatch = new CountDownLatch(1); var executorService = Executors.newFixedThreadPool(1); try { Future future = executorService.submit( () -> { log.info("Background thread starting"); txManager() .inTransactionThrows( tx -> { log.info("Background thread attempting select batch"); var batch = persistor().selectBatch(tx, 2, now); assertThat(batch, hasSize(2)); log.info("Background thread obtained locks, going to sleep"); gotLockLatch.countDown(); expectTobeInterrupted(); for (TransactionOutboxEntry entry : batch) { persistor().delete(tx, entry); } }); return null; }); // Wait for the background thread to have obtained the lock log.info("Waiting for background thread to obtain lock"); assertTrue(gotLockLatch.await(10, TimeUnit.SECONDS)); // Now try and select all four - we should only get two log.info("Attempting to obtain duplicate locks"); txManager() .inTransactionThrows( tx -> { var batch = persistor().selectBatch(tx, 4, now); assertThat(batch, hasSize(2)); for (TransactionOutboxEntry entry : batch) { persistor().delete(tx, entry); } }); // Kill the other thread log.info("Shutting down"); future.cancel(true); // Make sure any assertions from the other thread are propagated assertThrows(CancellationException.class, future::get); // Ensure that all the records are processed txManager() .inTransactionThrows(tx -> assertThat(persistor().selectBatch(tx, 100, now), empty())); } finally { executorService.shutdown(); assertTrue(executorService.awaitTermination(10, TimeUnit.SECONDS)); } } @Test public void testLockPessimisticLockFailure() throws Exception { TransactionOutboxEntry entry = createEntry("FOO1", now, false); txManager().inTransactionThrows(tx -> persistor().save(tx, entry)); CountDownLatch gotLockLatch = new CountDownLatch(1); ExecutorService executorService = Executors.newFixedThreadPool(1); try { // Submit another thread which will take a lock and hold it. If it is not // told to stop after 10 seconds it fails. Future future = executorService.submit( () -> { log.info("Background thread starting"); txManager() .inTransactionThrows( tx -> { log.info("Background thread attempting lock"); assertDoesNotThrow(() -> persistor().lock(tx, entry)); log.info("Background thread obtained lock, going to sleep"); gotLockLatch.countDown(); expectTobeInterrupted(); }); }); // Wait for the background thread to have obtained the lock log.info("Waiting for background thread to obtain lock"); assertTrue(gotLockLatch.await(10, TimeUnit.SECONDS)); // Now try and take the lock, which should fail log.info("Attempting to obtain duplicate lock"); txManager().inTransactionThrows(tx -> assertFalse(persistor().lock(tx, entry))); // Kill the other thread log.info("Shutting down"); future.cancel(true); // Make sure any assertions from the other thread are propagated assertThrows(CancellationException.class, future::get); } finally { executorService.shutdown(); assertTrue(executorService.awaitTermination(10, TimeUnit.SECONDS)); } } private TransactionOutboxEntry createEntry(String id, Instant nextAttemptTime, boolean blocked) { return TransactionOutboxEntry.builder() .id(id) .invocation(createInvocation()) .blocked(blocked) .lastAttemptTime(null) .nextAttemptTime(nextAttemptTime.truncatedTo(MILLIS)) .build(); } private TransactionOutboxEntry createEntry( String id, Instant nextAttemptTime, @SuppressWarnings("SameParameterValue") boolean blocked, String uniqueId) { return TransactionOutboxEntry.builder() .id(id) .invocation(createInvocation()) .blocked(blocked) .lastAttemptTime(null) .nextAttemptTime(nextAttemptTime.truncatedTo(MILLIS)) .uniqueRequestId(uniqueId) .build(); } private TransactionOutboxEntry createEntry( String id, Instant nextAttemptTime, Invocation invocation) { return TransactionOutboxEntry.builder() .id(id) .invocation(invocation) .blocked(false) .lastAttemptTime(null) .nextAttemptTime(nextAttemptTime.truncatedTo(MILLIS)) .build(); } private Invocation createInvocation() { return new Invocation( "Foo", "Bar", new Class[] {int.class, BigDecimal.class, String.class}, new Object[] {1, BigDecimal.TEN, null}); } private Invocation createUnparseableInvocation() { return new FailedDeserializingInvocation(new IOException()); } private void expectTobeInterrupted() { try { Thread.sleep(10000); throw new RuntimeException("Background thread not killed within 10 seconds"); } catch (InterruptedException e) { log.info("Background thread interrupted correctly"); } catch (Exception e) { log.error("Background thread failed", e); throw e; } } } ================================================ FILE: transactionoutbox-testing/src/main/java/com/gruelbox/transactionoutbox/testing/BaseTest.java ================================================ package com.gruelbox.transactionoutbox.testing; import com.gruelbox.transactionoutbox.*; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import java.sql.SQLException; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import lombok.Builder; import lombok.Value; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @Slf4j public abstract class BaseTest { protected HikariDataSource dataSource; private ExecutorService flushExecutor; @BeforeEach final void baseBeforeEach() { HikariConfig config = new HikariConfig(); config.setJdbcUrl(connectionDetails().url()); config.setUsername(connectionDetails().user()); config.setPassword(connectionDetails().password()); config.addDataSourceProperty("cachePrepStmts", "true"); dataSource = new HikariDataSource(config); flushExecutor = Executors.newFixedThreadPool(4); TestingMode.enable(); } @AfterEach final void baseAfterEach() throws InterruptedException { TestingMode.disable(); flushExecutor.shutdown(); Assertions.assertTrue(flushExecutor.awaitTermination(30, TimeUnit.SECONDS)); dataSource.close(); } protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(Dialect.H2) .driverClassName("org.h2.Driver") .url( "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DEFAULT_LOCK_TIMEOUT=60000;LOB_TIMEOUT=2000;MV_STORE=TRUE;DATABASE_TO_UPPER=FALSE") .user("test") .password("test") .build(); } protected TransactionManager txManager() { return TransactionManager.fromDataSource(dataSource); } protected Persistor persistor() { return Persistor.forDialect(connectionDetails().dialect()); } protected void clearOutbox() { DefaultPersistor persistor = Persistor.forDialect(connectionDetails().dialect()); TransactionManager transactionManager = txManager(); transactionManager.inTransaction( tx -> { try { persistor.clear(tx); } catch (SQLException e) { throw new RuntimeException(e); } }); } protected void withRunningFlusher(TransactionOutbox outbox, ThrowingRunnable runnable) throws Exception { withRunningFlusher(outbox, runnable, flushExecutor); } protected void withRunningFlusher( TransactionOutbox outbox, ThrowingRunnable runnable, Executor executor) throws Exception { withRunningFlusher(outbox, runnable, executor, null); } protected void withRunningFlusher( TransactionOutbox outbox, ThrowingRunnable runnable, Executor executor, String topicName) throws Exception { Thread backgroundThread = new Thread( () -> { while (!Thread.interrupted()) { try { // Keep flushing work until there's nothing left to flush log.info("Starting flush..."); while (topicName == null ? outbox.flush(executor) : outbox.flushTopics(executor, topicName)) { log.info("More work to do..."); } log.info("Done!"); } catch (Exception e) { log.error("Error flushing transaction outbox", e); } try { //noinspection BusyWait Thread.sleep(250); } catch (InterruptedException e) { break; } } }); backgroundThread.start(); try { runnable.run(); } finally { backgroundThread.interrupt(); backgroundThread.join(); } } @Value @Accessors(fluent = true) @Builder public static class ConnectionDetails { String driverClassName; String url; String user; String password; Dialect dialect; } } ================================================ FILE: transactionoutbox-testing/src/main/java/com/gruelbox/transactionoutbox/testing/ClassProcessor.java ================================================ package com.gruelbox.transactionoutbox.testing; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ClassProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(ClassProcessor.class); static final List PROCESSED = new CopyOnWriteArrayList<>(); void process(String itemId) { LOGGER.info("Processing work: {}", itemId); PROCESSED.add(itemId); } } ================================================ FILE: transactionoutbox-testing/src/main/java/com/gruelbox/transactionoutbox/testing/InterfaceProcessor.java ================================================ package com.gruelbox.transactionoutbox.testing; public interface InterfaceProcessor { void process(int foo, String bar); } ================================================ FILE: transactionoutbox-testing/src/main/java/com/gruelbox/transactionoutbox/testing/LatchListener.java ================================================ package com.gruelbox.transactionoutbox.testing; import com.gruelbox.transactionoutbox.TransactionOutboxEntry; import com.gruelbox.transactionoutbox.TransactionOutboxListener; import java.util.concurrent.CountDownLatch; import lombok.Getter; public final class LatchListener implements TransactionOutboxListener { private final CountDownLatch successLatch; private final CountDownLatch blockedLatch; @Getter private volatile TransactionOutboxEntry blocked; public LatchListener(CountDownLatch successLatch, CountDownLatch markFailedLatch) { this.successLatch = successLatch; this.blockedLatch = markFailedLatch; } public LatchListener(CountDownLatch successLatch) { this.successLatch = successLatch; this.blockedLatch = new CountDownLatch(1); } @Override public void success(TransactionOutboxEntry entry) { successLatch.countDown(); } @Override public void blocked(TransactionOutboxEntry entry, Throwable cause) { this.blocked = entry; blockedLatch.countDown(); } } ================================================ FILE: transactionoutbox-testing/src/main/java/com/gruelbox/transactionoutbox/testing/OrderedEntryListener.java ================================================ package com.gruelbox.transactionoutbox.testing; import com.gruelbox.transactionoutbox.TransactionOutboxEntry; import com.gruelbox.transactionoutbox.TransactionOutboxListener; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import lombok.Getter; import lombok.extern.slf4j.Slf4j; /** * Collects an ordered list of all entry events (*excluding blocked events) that have hit this * listener until a specified number of blocks / successes have occurred. */ @Slf4j public final class OrderedEntryListener implements TransactionOutboxListener { private final CountDownLatch successLatch; private final CountDownLatch blockedLatch; @Getter private volatile TransactionOutboxEntry blocked; private final CopyOnWriteArrayList events = new CopyOnWriteArrayList<>(); private final CopyOnWriteArrayList successes = new CopyOnWriteArrayList<>(); public OrderedEntryListener(CountDownLatch successLatch, CountDownLatch blockedLatch) { this.successLatch = successLatch; this.blockedLatch = blockedLatch; } @Override public void scheduled(TransactionOutboxEntry entry) { events.add(from(entry)); } @Override public void success(TransactionOutboxEntry entry) { var copy = from(entry); events.add(copy); successes.add(copy); log.info( "Received success #{}. Counting down at {}", successes.size(), successLatch.getCount()); successLatch.countDown(); } @Override public void failure(TransactionOutboxEntry entry, Throwable cause) { events.add(from(entry)); } @Override public void blocked(TransactionOutboxEntry entry, Throwable cause) { // due to the implementation of outbox (how it persists updates), it does not make sense to add // the blocked entry to the list for our current testing purposes. blocked = from(entry); blockedLatch.countDown(); } /** * Retrieve an unmodifiable copy of {@link #events}. Beware, expectation is that this does not/ * should not get accessed until the correct number of {@link #success(TransactionOutboxEntry)} * and {@link #blocked(TransactionOutboxEntry, Throwable)}} counts have occurred. * * @return unmodifiable list of ordered outbox entry events. */ public List getEvents() { return List.copyOf(events); } public List getSuccesses() { return List.copyOf(successes); } private TransactionOutboxEntry from(TransactionOutboxEntry entry) { return entry.toBuilder().build(); } } ================================================ FILE: transactionoutbox-testing/src/main/java/com/gruelbox/transactionoutbox/testing/ProcessedEntryListener.java ================================================ package com.gruelbox.transactionoutbox.testing; import com.gruelbox.transactionoutbox.TransactionOutboxEntry; import com.gruelbox.transactionoutbox.TransactionOutboxListener; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import lombok.extern.slf4j.Slf4j; /** * Collects an ordered list of tx outbox entries that have been 'processed' i.e. succeeded or failed * in processing. */ @Slf4j public final class ProcessedEntryListener implements TransactionOutboxListener { private final CountDownLatch successLatch; private final CopyOnWriteArrayList successfulEntries = new CopyOnWriteArrayList<>(); private final CopyOnWriteArrayList failingEntries = new CopyOnWriteArrayList<>(); public ProcessedEntryListener(CountDownLatch successLatch) { this.successLatch = successLatch; } @Override public void success(TransactionOutboxEntry entry) { var copy = from(entry); successfulEntries.add(copy); log.info( "Received success #{}. Counting down at {}", successfulEntries.size(), successLatch.getCount()); successLatch.countDown(); } @Override public void failure(TransactionOutboxEntry entry, Throwable cause) { failingEntries.add(from(entry)); } /** * Retrieve an unmodifiable copy of {@link #successfulEntries}. Beware, expectation is that this * does not/ should not get accessed until the correct number of {@link * #success(TransactionOutboxEntry)} and {@link #blocked(TransactionOutboxEntry, Throwable)}} * counts have occurred. * * @return unmodifiable list of ordered outbox entry events. */ public List getSuccessfulEntries() { return List.copyOf(successfulEntries); } public List getFailingEntries() { return List.copyOf(failingEntries); } private TransactionOutboxEntry from(TransactionOutboxEntry entry) { return entry.toBuilder().build(); } } ================================================ FILE: transactionoutbox-testing/src/main/java/com/gruelbox/transactionoutbox/testing/TestUtils.java ================================================ package com.gruelbox.transactionoutbox.testing; import com.gruelbox.transactionoutbox.TransactionManager; import java.sql.Statement; public class TestUtils { @SuppressWarnings("SameParameterValue") public static void runSql(TransactionManager transactionManager, String sql) { transactionManager.inTransaction( tx -> { try { try (Statement statement = tx.connection().createStatement()) { statement.execute(sql); } } catch (Exception e) { throw new RuntimeException(e); } }); } } ================================================ FILE: transactionoutbox-testing/src/main/resources/logback-test.xml ================================================ %d{HH:mm:ss.SSS} [%thread] %-5level %logger{5} - %msg%n ================================================ FILE: transactionoutbox-virtthreads/pom.xml ================================================ transactionoutbox-parent com.gruelbox ${revision} 4.0.0 Transaction Outbox Virtual Threads support jar transactionoutbox-virtthreads A safe implementation of the transactional outbox pattern for Java (core library) 25 25 com.gruelbox transactionoutbox-core ${project.version} test org.projectlombok lombok com.gruelbox transactionoutbox-testing ${project.version} test com.gruelbox transactionoutbox-jooq ${project.version} test org.testcontainers testcontainers org.testcontainers junit-jupiter org.testcontainers postgresql org.testcontainers oracle-xe org.testcontainers mysql org.postgresql postgresql com.oracle.database.jdbc ojdbc11 com.mysql mysql-connector-j com.h2database h2 org.testcontainers mssqlserver com.microsoft.sqlserver mssql-jdbc ================================================ FILE: transactionoutbox-virtthreads/src/test/java/com/gruelbox/transactionoutbox/virtthreads/AbstractVirtualThreadsTest.java ================================================ package com.gruelbox.transactionoutbox.virtthreads; import static java.util.stream.Collectors.joining; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import com.gruelbox.transactionoutbox.*; import com.gruelbox.transactionoutbox.testing.BaseTest; import com.gruelbox.transactionoutbox.testing.InterfaceProcessor; import java.lang.reflect.InvocationTargetException; import java.sql.SQLException; import java.time.Duration; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.IntStream; import jdk.jfr.consumer.RecordedEvent; import jdk.jfr.consumer.RecordingStream; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @Slf4j abstract class AbstractVirtualThreadsTest extends BaseTest { private static final String VIRTUAL_THREAD_SCHEDULER_PARALLELISM = "jdk.virtualThreadScheduler.parallelism"; protected RecordingStream stream; private final AtomicBoolean alerted = new AtomicBoolean(); private final CountDownLatch pinLatch = new CountDownLatch(1); @BeforeEach final void beforeEachAbstractVirtualThreadsTest() throws InterruptedException { stream = new RecordingStream(); stream.enable("jdk.VirtualThreadPinned").withThreshold(Duration.ZERO); stream.onEvent( "jdk.VirtualThreadPinned", event -> { if (isTransientInitialization(event)) { log.info("Ignored classloader issues since these are transient..."); return; } log.info( """ Pinning Event Captured: Reason: %s Blocking operation: %s Duration: %dms Stack trace: %s""" .formatted( event.getValue("pinnedReason"), event.getValue("blockingOperation"), event.getDuration().toMillis(), event.getStackTrace().getFrames().stream() .map( f -> " - %s %s.%s(%s)" .formatted( f.getType(), f.getMethod().getType().getName(), f.getMethod().getName(), f.getLineNumber())) .collect(joining("\n")))); pinLatch.countDown(); }); stream.startAsync(); Thread.sleep(500); } private boolean isTransientInitialization(RecordedEvent event) { var stackTrace = event.getStackTrace(); if (stackTrace == null) return false; return stackTrace.getFrames().stream() .anyMatch( frame -> { String method = frame.getMethod().getType().getName(); return method.contains("jdk.internal.loader") || method.contains("java.lang.ClassLoader") || method.contains("java.lang.invoke.MethodHandles"); }); } @AfterEach final void afterEachAbstractVirtualThreadsTest() { try { if (didPin()) { fail("Virtual thread was pinned. See earlier messages"); } } finally { stream.close(); } } protected boolean didPin() { try { if (alerted.getAndSet(true)) { log.info("Suppressed pinning alert as already checked"); return false; } if (!pinLatch.await(5, TimeUnit.SECONDS)) { return false; } return true; } catch (InterruptedException e) { throw new RuntimeException(e); } } @Test final void highVolumeVirtualThreads() throws Exception { var count = 10; var latch = new CountDownLatch(count * 10); var transactionManager = txManager(); var results = new ConcurrentHashMap(); var duplicates = new ConcurrentHashMap(); var persistor = Persistor.forDialect(connectionDetails().dialect()); var outbox = TransactionOutbox.builder() .transactionManager(transactionManager) .persistor(persistor) .instantiator(Instantiator.using(clazz -> (InterfaceProcessor) (foo, bar) -> {})) .submitter( Submitter.withExecutor( r -> Thread.ofVirtual().name(UUID.randomUUID().toString()).start(r))) .attemptFrequency(Duration.ofMillis(500)) .flushBatchSize(1000) .listener( new TransactionOutboxListener() { @Override public void success(TransactionOutboxEntry entry) { Integer i = (Integer) entry.getInvocation().getArgs()[0]; if (results.putIfAbsent(i, i) != null) { duplicates.put(i, i); } latch.countDown(); } }) .build(); warmupJdk(transactionManager, persistor); var parallelism = System.getProperty(VIRTUAL_THREAD_SCHEDULER_PARALLELISM); System.setProperty(VIRTUAL_THREAD_SCHEDULER_PARALLELISM, "1"); try { withRunningFlusher( outbox, () -> { var futures = IntStream.range(0, count) .mapToObj( i -> new FutureTask( () -> transactionManager.inTransaction( () -> { for (int j = 0; j < 10; j++) { outbox .schedule(InterfaceProcessor.class) .process(i * 10 + j, "Whee"); } }), null)) .toList(); futures.forEach(Thread::startVirtualThread); for (var future : futures) { future.get(20, TimeUnit.SECONDS); } assertTrue(latch.await(30, TimeUnit.SECONDS), "Latch not opened in time"); }); } finally { if (parallelism == null) { System.clearProperty(VIRTUAL_THREAD_SCHEDULER_PARALLELISM); } else { System.setProperty(VIRTUAL_THREAD_SCHEDULER_PARALLELISM, parallelism); } } assertThat( "Should never get duplicates running to full completion", duplicates.keySet(), empty()); assertThat( "Only got: " + results.keySet(), results.keySet(), containsInAnyOrder(IntStream.range(0, count * 10).boxed().toArray())); } private void warmupJdk(TransactionManager transactionManager, DefaultPersistor persistor) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, SQLException { // Warm up Invocation.invoke so it converts to a MethodHandle now rather than later (which // causes pinning) for (int i = 0; i < 50; i++) { var invocation = new Invocation( InterfaceProcessor.class.getName(), "process", new Class[] {int.class, String.class}, new Object[] {1, ""}); InterfaceProcessor warmupTarget = (foo, bar) -> { log.info("Warmup"); }; var invokeMethod = Invocation.class.getDeclaredMethod( "invoke", Object.class, TransactionOutboxListener.class); invokeMethod.setAccessible(true); invokeMethod.invoke(invocation, warmupTarget, new TransactionOutboxListener() {}); } // And do a bit of database access now to warm up the JDBC driver transactionManager.inTransactionThrows( tx -> { Assert.assertTrue(persistor.checkConnection(tx)); persistor.clear(tx); }); } } ================================================ FILE: transactionoutbox-virtthreads/src/test/java/com/gruelbox/transactionoutbox/virtthreads/TestVirtualThreadsH2.java ================================================ package com.gruelbox.transactionoutbox.virtthreads; import static org.junit.Assert.assertTrue; import java.lang.foreign.*; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; @Slf4j public class TestVirtualThreadsH2 extends AbstractVirtualThreadsTest { private static final String QSORT = "qsort"; /** * Ensures that the logic we use in {@link AbstractVirtualThreadsTest} to detect thread pinning * actually works. In Java 25 we have to turn ourselves inside out to achieve this; even {@code * synchronized} doesn't pin a thread anymore. The only thing that seems to work is to make a * native call that calls back to Java. When that stops working, this test can probably be * removed; there will be very little likelihood of pinning in practice once that last thing is * resolved in the JDK. */ @Test void forceTriggerPinningViaUpcall() throws Throwable { simulateThreadPin(); // Prevents the check on exit from throwing assertTrue(didPin()); } private void simulateThreadPin() throws NoSuchMethodException, IllegalAccessException { Linker linker = Linker.nativeLinker(); SymbolLookup libc = name -> { try { var lookup = System.getProperty("os.name").toLowerCase().contains("win") ? SymbolLookup.libraryLookup("msvcrt.dll", Arena.global()) : SymbolLookup.libraryLookup("libc.so.6", Arena.global()); return lookup.find(name); } catch (Exception e) { return Optional.empty(); } }; MethodHandle qsort = linker.downcallHandle( libc.find(QSORT) .or(() -> Linker.nativeLinker().defaultLookup().find(QSORT)) .or(() -> SymbolLookup.loaderLookup().find(QSORT)) .orElseThrow(() -> new RuntimeException("Could not find qsort")), FunctionDescriptor.ofVoid( ValueLayout.ADDRESS, ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS)); MethodHandle upcallTarget = MethodHandles.lookup() .findStatic( getClass(), "javaBlockCallback", MethodType.methodType(int.class, MemorySegment.class, MemorySegment.class)); MemorySegment upcallStub = linker.upcallStub( upcallTarget, FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.ADDRESS, ValueLayout.ADDRESS), Arena.global()); Thread.ofVirtual() .name("Pin-Me-Thread") .start( () -> { try { System.out.println("Virtual thread entering native qsort..."); MemorySegment data = Arena.ofConfined().allocate(16); // 2 long elements qsort.invoke(data, 2L, 8L, upcallStub); System.out.println("Virtual thread exited native qsort."); } catch (Throwable e) { throw new RuntimeException(e); } }); } // This is the Upcall: Java -> Native (qsort) -> Java (here) public static int javaBlockCallback(MemorySegment a, MemorySegment b) { try { // This 'park' operation happens while a native frame (qsort) // is on the stack. Pinning is guaranteed. Thread.sleep(200); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } return 0; } } ================================================ FILE: transactionoutbox-virtthreads/src/test/java/com/gruelbox/transactionoutbox/virtthreads/TestVirtualThreadsH2Jooq.java ================================================ package com.gruelbox.transactionoutbox.virtthreads; import com.gruelbox.transactionoutbox.ThreadLocalContextTransactionManager; import com.gruelbox.transactionoutbox.jooq.JooqTransactionListener; import com.gruelbox.transactionoutbox.jooq.JooqTransactionManager; import org.jooq.SQLDialect; import org.jooq.impl.DSL; import org.jooq.impl.DataSourceConnectionProvider; import org.jooq.impl.DefaultConfiguration; import org.jooq.impl.ThreadLocalTransactionProvider; import org.junit.jupiter.api.BeforeEach; public class TestVirtualThreadsH2Jooq extends AbstractVirtualThreadsTest { private ThreadLocalContextTransactionManager txm; @Override protected final ThreadLocalContextTransactionManager txManager() { return txm; } @BeforeEach final void beforeEach() { DataSourceConnectionProvider connectionProvider = new DataSourceConnectionProvider(dataSource); DefaultConfiguration configuration = new DefaultConfiguration(); configuration.setConnectionProvider(connectionProvider); configuration.setSQLDialect(SQLDialect.H2); configuration.setTransactionProvider( new ThreadLocalTransactionProvider(connectionProvider, true)); JooqTransactionListener listener = JooqTransactionManager.createListener(); configuration.set(listener); txm = JooqTransactionManager.create(DSL.using(configuration), listener); } } ================================================ FILE: transactionoutbox-virtthreads/src/test/java/com/gruelbox/transactionoutbox/virtthreads/TestVirtualThreadsMySql5.java ================================================ package com.gruelbox.transactionoutbox.virtthreads; import static com.gruelbox.transactionoutbox.Dialect.MY_SQL_5; import java.time.Duration; import java.util.Map; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestVirtualThreadsMySql5 extends AbstractVirtualThreadsTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = new MySQLContainer<>("mysql:5") .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true) .withTmpFs(Map.of("/var/lib/mysql", "rw")); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(MY_SQL_5) .driverClassName("com.mysql.cj.jdbc.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-virtthreads/src/test/java/com/gruelbox/transactionoutbox/virtthreads/TestVirtualThreadsMySql8.java ================================================ package com.gruelbox.transactionoutbox.virtthreads; import static com.gruelbox.transactionoutbox.Dialect.MY_SQL_8; import java.time.Duration; import java.util.Map; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestVirtualThreadsMySql8 extends AbstractVirtualThreadsTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = new MySQLContainer<>("mysql:8") .withStartupTimeout(Duration.ofMinutes(5)) .withReuse(true) .withTmpFs(Map.of("/var/lib/mysql", "rw")); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(MY_SQL_8) .driverClassName("com.mysql.cj.jdbc.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-virtthreads/src/test/java/com/gruelbox/transactionoutbox/virtthreads/TestVirtualThreadsOracle21.java ================================================ package com.gruelbox.transactionoutbox.virtthreads; import static com.gruelbox.transactionoutbox.Dialect.ORACLE; import java.time.Duration; import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.OracleContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers @Disabled // Struggling to avoid pinning here at the moment class TestVirtualThreadsOracle21 extends AbstractVirtualThreadsTest { @Container @SuppressWarnings("rawtypes") private static final JdbcDatabaseContainer container = new OracleContainer("gvenzl/oracle-xe:21-slim-faststart") .withStartupTimeout(Duration.ofHours(1)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(ORACLE) .driverClassName("oracle.jdbc.OracleDriver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-virtthreads/src/test/java/com/gruelbox/transactionoutbox/virtthreads/TestVirtualThreadsPostgres16.java ================================================ package com.gruelbox.transactionoutbox.virtthreads; import static com.gruelbox.transactionoutbox.Dialect.POSTGRESQL_9; import java.time.Duration; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @SuppressWarnings("WeakerAccess") @Testcontainers class TestVirtualThreadsPostgres16 extends AbstractVirtualThreadsTest { @Container @SuppressWarnings({"rawtypes", "resource"}) private static final JdbcDatabaseContainer container = (JdbcDatabaseContainer) new PostgreSQLContainer("postgres:16") .withStartupTimeout(Duration.ofHours(1)) .withReuse(true); @Override protected ConnectionDetails connectionDetails() { return ConnectionDetails.builder() .dialect(POSTGRESQL_9) .driverClassName("org.postgresql.Driver") .url(container.getJdbcUrl()) .user(container.getUsername()) .password(container.getPassword()) .build(); } } ================================================ FILE: transactionoutbox-virtthreads/src/test/resources/logback-test.xml ================================================ %d{HH:mm:ss.SSS} [%thread] %-5level %logger{5} - %msg%n ================================================ FILE: ~/settings.xml ================================================ github ${env.GITHUB_ACTOR} ${env.GITHUB_TOKEN} ================================================ FILE: ~/toolchains.xml ================================================ jdk 25 zulu zulu_25 /opt/hostedtoolcache/Java_Zulu_jdk/25.0.2-10/x64