Repository: klarna/HiveRunner Branch: main Commit: 0be34c87c421 Files: 184 Total size: 486.2 KB Directory structure: gitextract_l_9r6xof/ ├── .github/ │ ├── CODEOWNERS │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ └── feature_request.md │ └── workflows/ │ ├── deploy.yml │ ├── main.yml │ └── release.yml ├── .gitignore ├── CHANGELOG.md ├── CODE-OF-CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── RELEASING.md ├── pom.xml └── src/ ├── main/ │ ├── java/ │ │ └── com/ │ │ └── klarna/ │ │ ├── hiverunner/ │ │ │ ├── HiveRunnerCore.java │ │ │ ├── HiveRunnerExtension.java │ │ │ ├── HiveRunnerRule.java │ │ │ ├── HiveServerContainer.java │ │ │ ├── HiveServerContext.java │ │ │ ├── HiveShell.java │ │ │ ├── HiveShellContainer.java │ │ │ ├── StandaloneHiveRunner.java │ │ │ ├── StandaloneHiveServerContext.java │ │ │ ├── ThrowOnTimeout.java │ │ │ ├── TimeoutException.java │ │ │ ├── annotations/ │ │ │ │ ├── HiveProperties.java │ │ │ │ ├── HiveResource.java │ │ │ │ ├── HiveRunnerSetup.java │ │ │ │ ├── HiveSQL.java │ │ │ │ └── HiveSetupScript.java │ │ │ ├── builder/ │ │ │ │ ├── HiveResource.java │ │ │ │ ├── HiveRunnerScript.java │ │ │ │ ├── HiveShellBase.java │ │ │ │ ├── HiveShellBuilder.java │ │ │ │ ├── HiveShellTearable.java │ │ │ │ ├── Script.java │ │ │ │ └── Statement.java │ │ │ ├── config/ │ │ │ │ └── HiveRunnerConfig.java │ │ │ ├── data/ │ │ │ │ ├── Converters.java │ │ │ │ ├── FileParser.java │ │ │ │ ├── InsertIntoTable.java │ │ │ │ ├── TableDataBuilder.java │ │ │ │ ├── TableDataInserter.java │ │ │ │ └── TsvFileParser.java │ │ │ ├── io/ │ │ │ │ └── IgnoreClosePrintStream.java │ │ │ └── sql/ │ │ │ ├── HiveRunnerStatement.java │ │ │ ├── StatementLexer.java │ │ │ ├── cli/ │ │ │ │ ├── AbstractImportPostProcessor.java │ │ │ │ ├── CommandShellEmulator.java │ │ │ │ ├── CommandShellEmulatorFactory.java │ │ │ │ ├── CommentUtil.java │ │ │ │ ├── DefaultPreProcessor.java │ │ │ │ ├── PostProcessor.java │ │ │ │ ├── PreProcessor.java │ │ │ │ ├── beeline/ │ │ │ │ │ ├── BeelineEmulator.java │ │ │ │ │ ├── RunCommandPostProcessor.java │ │ │ │ │ └── SqlLineCommandRule.java │ │ │ │ └── hive/ │ │ │ │ ├── HiveCliEmulator.java │ │ │ │ ├── PreV200HiveCliEmulator.java │ │ │ │ ├── PreV200HiveCliPreProcessor.java │ │ │ │ └── SourceCommandPostProcessor.java │ │ │ └── split/ │ │ │ ├── BaseContext.java │ │ │ ├── CloseStatementRule.java │ │ │ ├── Consumer.java │ │ │ ├── Context.java │ │ │ ├── DefaultTokenRule.java │ │ │ ├── NewLineUtil.java │ │ │ ├── PreserveCommentsRule.java │ │ │ ├── PreserveQuotesRule.java │ │ │ ├── StatementSplitter.java │ │ │ └── TokenRule.java │ │ └── reflection/ │ │ └── ReflectionUtils.java │ └── license/ │ └── APACHE-2.txt └── test/ ├── java/ │ └── com/ │ └── klarna/ │ └── hiverunner/ │ ├── AggregateViewTest.java │ ├── AnnotatedBaseTestClass.java │ ├── AnnotatedFieldsInSuperClassTest.java │ ├── BeelineRunTest.java │ ├── BigResultSetTest.java │ ├── CommentTest.java │ ├── CtasTest.java │ ├── DisabledTimeoutTest.java │ ├── ExecuteFileBasedScriptIntegrationTest.java │ ├── ExecuteScriptIntegrationTest.java │ ├── HiveCliSourceTest.java │ ├── HiveRunnerAnnotationsTest.java │ ├── HiveRunnerExtensionTest.java │ ├── HiveServerContainerTest.java │ ├── HiveShellBeeLineEmulationTest.java │ ├── HiveShellHiveCliEmulationTest.java │ ├── HiveVariablesTest.java │ ├── InsertIntoTableIntegrationTest.java │ ├── IntegerPartitionFormatTest.java │ ├── InteractiveHiveShellTest.java │ ├── LeftOuterJoinTest.java │ ├── MSCKRepairNpeTest.java │ ├── MacroTest.java │ ├── MethodLevelResourceTest.java │ ├── MultipleExecutionEnginesTest.java │ ├── NeverEndingUdf.java │ ├── NoTimeoutTest.java │ ├── OrcSnappyTest.java │ ├── ParquetInsertionTest.java │ ├── PartitionSupportTest.java │ ├── ReservedKeywordTest.java │ ├── ResourceOutputStreamTest.java │ ├── SchemaResetBetweenTestMethodsTest.java │ ├── SerdeTest.java │ ├── SetHiveExecutionEngineTest.java │ ├── SetPropertyTest.java │ ├── SetTest.java │ ├── SlowlyFailingUdf.java │ ├── TestMethodIntegrityTest.java │ ├── TimeoutAndRetryTest.java │ ├── ToUpperCaseSerDe.java │ ├── UnresolvedResourcePathTest.java │ ├── UserDefinedFunctionTest.java │ ├── builder/ │ │ └── HiveShellBaseTest.java │ ├── config/ │ │ └── HiveRunnerConfigTest.java │ ├── data/ │ │ ├── ConvertersTest.java │ │ ├── InsertIntoTableTest.java │ │ ├── TableDataBuilderTest.java │ │ ├── TableDataInserterTest.java │ │ └── TsvFileParserTest.java │ ├── examples/ │ │ ├── HelloAnnotatedHiveRunnerTest.java │ │ ├── HelloHiveRunnerParamaterizedTest.java │ │ ├── HelloHiveRunnerTest.java │ │ ├── InsertTestDataTest.java │ │ ├── SetHiveConfValuesTest.java │ │ └── junit4/ │ │ ├── HelloAnnotatedHiveRunnerTest.java │ │ ├── HelloHiveRunnerTest.java │ │ ├── InsertTestDataTest.java │ │ └── SetHiveConfValuesTest.java │ ├── io/ │ │ └── IgnoreClosePrintStreamTest.java │ └── sql/ │ ├── cli/ │ │ ├── AbstractImportPostProcessorTest.java │ │ ├── CommandShellEmulatorFactoryTest.java │ │ ├── CommentUtilTest.java │ │ ├── beeline/ │ │ │ ├── BeelineEmulatorTest.java │ │ │ ├── BeelineStatementSplitterTest.java │ │ │ ├── RunCommandPostProcessorTest.java │ │ │ └── SqlLineCommandRuleTest.java │ │ └── hive/ │ │ ├── HiveCliEmulatorTest.java │ │ ├── HiveCliStatementSplitterTest.java │ │ ├── PreV200HiveCliEmulatorTest.java │ │ └── SourceCommandPostProcessorTest.java │ └── split/ │ ├── BaseContextTest.java │ ├── CloseStatementRuleTest.java │ ├── ConsumerEolTest.java │ ├── DefaultTokenRuleTest.java │ ├── NewLineUtilTest.java │ ├── PreserveCommentsRuleTest.java │ ├── PreserveQuotesRuleTest.java │ └── StatementSplitterTest.java └── resources/ ├── AggregateViewTest/ │ └── create_table.sql ├── CommentTest/ │ └── comment.sql ├── CtasTest/ │ └── ctas.sql ├── HelloHiveRunnerTest/ │ ├── calculate_max.sql │ ├── create_ctas.sql │ ├── create_max.sql │ ├── create_table.sql │ └── hello_hive_runner.csv ├── HiveRunnerAnnotationsTest/ │ ├── hql1.sql │ ├── setupFile.csv │ ├── setupPath.csv │ ├── testData.csv │ └── testData2.csv ├── HiveRunnerExtensionTest/ │ └── test_query.sql ├── InsertIntoTableIntegrationTest/ │ ├── data.tsv │ └── dataWithCustomNullValue.csv ├── InsertTestDataTest/ │ ├── data1.tsv │ ├── data2.tsv │ ├── dataWithHeader1.tsv │ └── dataWithHeader2.tsv ├── MethodLevelResourceTest/ │ └── MethodLevelResourceTest.txt ├── OrcSnappyTest/ │ └── ctas.sql ├── PartitionSupportTest/ │ └── hql_example.sql ├── SerdeTest/ │ ├── create_table.sql │ └── hql_custom_serde.sql ├── SetTest/ │ └── test_with_set.hql ├── TsvFileParserTest/ │ ├── data.csv │ ├── data.tsv │ ├── dataWithCustomNullValue.csv │ ├── dataWithHeader.csv │ └── dataWithHeader.tsv └── log4j2.xml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/CODEOWNERS ================================================ * @HiveRunner/hiverunner-committers ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug report about: Create a report to help us improve --- **Describe the bug** A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behaviour ideally including the configuration files you are using (feel free to rename any sensitive information like server and table names etc.) Even better would be the source code of your unit test or a pull request against the HiveRunner unit tests containing a test that demonstrates the issue. **Expected behavior** A clear and concise description of what you expected to happen. **Logs** Please add the log output from HiveRunner when the error occurs, full stack traces are especially useful. **Versions (please complete the following information):** - HiveRunner Version: - Hive Versions: for whatever version of Hive you are using for your tests **Additional context** Add any other context about the problem here. ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request about: Suggest an idea for this project --- **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. For example - I'm always frustrated when [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. **Additional context** Add any other context or screenshots about the feature request here. ================================================ FILE: .github/workflows/deploy.yml ================================================ name: Deploy SNAPSHOT on: workflow_dispatch: inputs: branch: description: "The branch to use to deploy a SNAPSHOT from." required: true default: "main" jobs: deploy: name: Deploy SNAPSHOT to Sonatype runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 with: fetch-depth: 0 ref: ${{ github.event.inputs.branch }} - name: Set up JDK uses: actions/setup-java@v2 with: distribution: 'adopt' java-version: '8' # this creates a settings.xml with the following server settings-path: ${{ github.workspace }} server-id: ossrh # Value of the distributionManagement/repository/id field of the pom.xml server-username: SONATYPE_USERNAME # env variable for username in deploy server-password: SONATYPE_PASSWORD # env variable for token in deploy # only signed artifacts will be released to maven central. this sets up things for the maven-gpg-plugin gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} # Value of the GPG private key to import gpg-passphrase: GPG_PASSPHRASE # env variable for GPG private key passphrase - name: Run Maven Targets run: mvn deploy --settings $GITHUB_WORKSPACE/settings.xml --batch-mode --show-version --no-transfer-progress --activate-profiles oss-release env: SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} GPG_PASSPHRASE: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }} ================================================ FILE: .github/workflows/main.yml ================================================ name: build on: pull_request: push: branches: - main jobs: test: name: Package and run all tests runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - name: Set up JDK uses: actions/setup-java@v2 with: distribution: 'adopt' java-version: '8' - name: Run Maven Targets run: mvn package --batch-mode --show-version --no-transfer-progress ================================================ FILE: .github/workflows/release.yml ================================================ name: Release to Maven Central on: workflow_dispatch: inputs: branch: description: "The branch to use to release from." required: true default: "main" jobs: release: name: Release to Maven Central runs-on: ubuntu-20.04 steps: - name: Checkout source code uses: actions/checkout@v2 with: fetch-depth: 0 ref: ${{ github.event.inputs.branch }} - name: Set up JDK uses: actions/setup-java@v2 with: distribution: 'adopt' java-version: '8' # this creates a settings.xml with the following server settings-path: ${{ github.workspace }} server-id: ossrh # Value of the distributionManagement/repository/id field of the pom.xml server-username: SONATYPE_USERNAME # env variable for username in deploy server-password: SONATYPE_PASSWORD # env variable for token in deploy # only signed artifacts will be released to maven central. this sets up things for the maven-gpg-plugin gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} # Value of the GPG private key to import gpg-passphrase: GPG_PASSPHRASE # env variable for GPG private key passphrase - name: Configure Git User run: | git config user.email "actions@github.com" git config user.name "GitHub Actions" - name: Run Maven Targets run: mvn release:prepare release:perform --settings $GITHUB_WORKSPACE/settings.xml --activate-profiles oss-release --batch-mode --show-version --no-transfer-progress env: SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} GPG_PASSPHRASE: ${{secrets.GPG_PRIVATE_KEY_PASSPHRASE}} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .gitignore ================================================ # Project files # ################# *.iml *.ipr *.iws nbactions.xml /.idea/ # Compiled source # ################### *.com *.class *.dll *.exe *.o *.so # Maven target # ################ /target/** # H2 db files # ############### mem.h2.db mem.lock.db # HSQLDB files # ################ testdb.log testdb.properties testdb.script # Packages # ############ # it's better to unpack these files and commit the raw source # git has its own built in compression methods *.7z *.dmg *.gz *.iso *.jar *.rar *.tar *.zip # Logs and databases # ###################### *.log *.sqlite metastore_db # OS generated files # ###################### .DS_Store .DS_Store? ._* .Spotlight-V100 .Trashes Icon? ehthumbs.db Thumbs.db # Temporary files # ################### *~ # Eclipse # ########### .classpath .project .settings/ # IntelliJ # ############ .gradle/ *.eml # jEnv # ######## /.java-version # Misc # ######## /pubring.kbx ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [7.0.0] - 2024-11-22 ### Added - Added version `6.0.9` of `datanucleus-core`. - Added version `6.0.4` of `datanucleus-api-jdo`. - Added version `6.0.9` of `datanucleus-rdbms`. - Added version `1.3` of `javax.transaction-api`. - Added version `6.1.14` of `spring-jdbc`. - Added version `10.15.2.0` of `derby`. - Added version `10.15.2.0` of `derbytools`. - Added version `5.6.2` of `kryo`. - Added version `4.9.3` of `antlr4-runtime`. - Added version `4.0.1` of `kafka-handler`. - Added missing Hive & Datanucleus properties in StandaloneHiveServerContext so now the framework works with a new Hive dependency versions. ### Changed - Updated `hadoop-mapreduce-client-common` from `3.1.0` to `3.4.1`. - Updated `hadoop-mapreduce-client-core` from `3.1.0` to `3.4.1`. - Updated `hadoop-client-runtime` from `3.1.0` to `3.4.1`. - Updated `hive-exec` from `3.1.2` to `4.0.1`. - Updated `hive-serde` from `3.1.0` to `3.4.1`. - Updated `hive-jdbc` from `3.1.0` to `3.4.1`. - Updated `hive-contrib` from `3.1.0` to `3.4.1`. - Updated `hive-webhcat-java-client` from `3.1.0` to `3.4.1`. - Updated `jackson-annotations` from `2.9.5` to `2.18.1`. - Updated `reflections` from `0.9.8` to `0.10.2`. - Updated `mockito-core` from `3.8.0` to `5.14.2`. - Updated `mockito-junit-jupiter` from `3.8.0` to `5.14.2`. - Updated `tez-common` from `0.9.1` to `0.10.4`. - Updated `tez-mapreduce` from `0.9.1` to `0.10.4`. - Updated `junit-jupiter` from `5.7.1` to `5.11.3`. - Updated `junit-vintage-engine` from `5.7.1` to `5.11.2`. - Updated `maven-surefire-plugin` from `2.22.2` to `3.5.1`. - Updated `maven-compiler-plugin` from `3.7.0` to `3.13.0`. - Updated `maven-jar-plugin` from `3.2.0` to `3.4.2`. - Updated `maven-release-plugin` from `3.0.0-M1` to `3.1.1`. - Updated `nexus-staging-maven-plugin` from `1.6.8` to `1.7.0`. - Updated `maven-source-plugin` from `3.2.0` to `3.3.1`. - Updated `maven-javadoc-plugin` from `3.2.0` to `3.10.1`. - Updated `maven-gpg-plugin` from `1.6` to `3.2.7`. - Updated `HiveConf` property names in `StandaloneHiveServerContext` - Set `METASTORE_VALIDATE_CONSTRAINTS`, `METASTORE_VALIDATE_COLUMNS`, `METASTORE_VALIDATE_TABLES` properties to false in StandaloneHiveServerContext. ### Removed - Removed `com.google.common.base.Predicates` in `HiveRunnerExtension`/`StandaloneHiveRunner` as it is no longer used in a new version of `org.reflections:reflections` library. ### Fixed - Fixed warning "org.apache.hadoop.hive.metastore.MetastoreDirectSqlUtils - Failed to execute [select "FUNCS"."FUNC_ID" from "FUNCS" LEFT JOIN "DBS" ON "FUNCS"."DB_ID" = "DBS"."DB_ID" where "DBS"."CTLG_NAME" = ? ]..." is not logged anymore. - Fixed `IgnoreClosePrintStream` as NPE was thrown after upgrading to Java >= 11 - Fixed error "org.apache.hadoop.hive.ql.exec.tez.DagUtils - Failed to add credential supplier java.lang.ClassNotFoundException: org.apache.hadoop.hive.kafka.KafkaDagCredentialSupplier" ## [6.1.0] - 2021-04-28 ### Changed - Maven Group Id changed from `com.klarna` to `io.github.hiverunner`. - Set `HIVE_IN_TEST` to true in `StandaloneHiverServerContext` instead of `StandaloneHiveRunner` so checks for non-existent tables are skipped by both the JUnit4 runner and the JUnit5 extension (this removes a lot of log noise from tests using the latter). - Made `HiveRunnerScript` constructor public. - Made `scriptsUnderTest` variable in `HiveRunnerExtension` protected so it can be used in [MutantSwarm](https://github.com/HotelsDotCom/mutant-swarm). - Fixed bug that appears in [Mutant Swarm](https://github.com/HotelsDotCom/mutant-swarm) when updating HiveRunner to version 5.2.1. - Renamed `HelloAnnotatedHiveRunner` in `com.klarna.hiverunner.examples` to `HelloAnnotatedHiveRunnerTest`. - Renamed `HelloHiveRunner` in `com.klarna.hiverunner.examples` to `HelloHiveRunnerTest`. - Renamed `InsertTestData` in `com.klarna.hiverunner.examples` to `InsertTestDataTest`. - Renamed `SetHiveConfValues` in `com.klarna.hiverunner.examples` to `SetHiveConfValuesTest`. - Renamed `HelloAnnotatedHiveRunner` in `com.klarna.hiverunner.examples.junit4` to `HelloAnnotatedHiveRunnerTest`. - Renamed `HelloHiveRunner` in `com.klarna.hiverunner.examples.junit4` to `HelloHiveRunnerTest`. - Renamed `InsertTestData` in `com.klarna.hiverunner.examples.junit4` to `InsertTestDataTest`. - Renamed `SetHiveConfValues` in `com.klarna.hiverunner.examples.junit4` to `SetHiveConfValuesTest`. - Updated `surefire-version-plugin` from `2.21.0` to `2.22.0`. - Updated `junit.jupiter.version` (JUnit5) from `5.6.0` to `5.7.1`. - Updated `junit` (JUnit4) from `4.13.1` to `4.13.2`. - Updated `mockito-core` from `2.18.3` to `3.8.0`. ### Added - Added `getScriptPaths` method in `HiveRunnerCore`. - Added `getScriptPaths` method in `HiveRunnerExtension` to be able to access the other method in `HiveRunnerCore` so that it can be used downstream in [MutantSwarm](https://github.com/HotelsDotCom/mutant-swarm). - Added `fromScriptPaths` method in `HiveShellBuilder`. - Added version `5.7.1` of `junit-vintage-engine`. - Added version `3.8.0` of `mockito-junit-jupiter`. ### Fixed - Fixed bug where the files specified in `@HiveSQL` weren't being run when using `HiveRunnerExtension`. - Successful tests using "SET" no longer marked as "terminated" when run in IntelliJ. See [#94](https://github.com/klarna/HiveRunner/issues/94). ## [6.0.1] - 2020-09-07 ### Removed - Removed shaded jar that was being produced as a side-effect. ## [6.0.0] - 2020-09-03 ### Changed - Upgraded Hive version to 3.1.2 (was 2.3.7). ## [5.x] ### NOTE - Releases from the 5.x (Hive 2) line are not tracked in this CHANGELOG, it only tracks 6.0.0 and above. For changes in 5.x please refer to https://github.com/klarna/HiveRunner/blob/hive-2.x/CHANGELOG.md. ## [5.0.0] - 2019-09-30 ### Added - JUnit5 [Extension](https://junit.org/junit5/docs/current/user-guide/#extensions) support with `HiveRunnerExtension`. See [#106](https://github.com/klarna/HiveRunner/issues/106). ### Changed - Default supported Hive version is now 2.3.4 (was 2.3.3) as version 2.3.3 has a [vulnerability](https://nvd.nist.gov/vuln/detail/CVE-2018-1314). - `TemporaryFolder` ([JUnit 4](https://junit.org/junit4/javadoc/4.12/org/junit/rules/TemporaryFolder.html)) has been changed to `Path` ([Java NIO](https://docs.oracle.com/javase/8/docs/api/java/nio/file/Path.html)) throughout the project for the JUnit5 update. - NOTE: The `HiveServerContext` class now uses `Path` instead of `TemporaryFolder` in the constructor. ## [4.1.0] - 2019-02-27 ### Changed - Internal refactoring to support upcoming "Mutant Swarm" project which provides unit test coverage for Hive SQL scripts. See [#65](https://github.com/klarna/HiveRunner/issues/65). ## [4.0.0] - 2018-07-17 ### Added - Support shell-specific `source` (`hive`) and ``!run`` (`beeline`) commands. These commands allow one to import and execute the contents of external files in statements or scripts. ### Changed - Default supported Hive version is now 2.3.3 (was 1.2.1). - Default supported Tez version is now 0.9.1 (was 0.7.0). - Supported Java version is 8 (was 7). - In-memory DB used by HiveRunner is now Derby (was HSQLDB). - Log4J configuration file removed from jar artifact. - System property to configure command shell emulation mode renamed to `commandShellEmulator` (was `commandShellEmulation`). ## [3.2.1] - 2018-05-31 ### Changed - Fixed issue where if case of column name in a file was different to case in table definition they would be treated as different [#73](https://github.com/klarna/HiveRunner/issues/73). - The way of setting writable permissions on JUnit temporary folder changed to make it compatible with Windows [#63](https://github.com/klarna/HiveRunner/issues/63). ## [3.2.0] - 2017-02-09 ### Added - Added functionality for headers in TSV parser. This way you can dynamically add TSV files declaring a subset of columns using insertInto. ## [3.1.1] - 2017-01-27 ### Added - Added debug logging of result set. Enable by setting ```log4j.logger.com.klarna.hiverunner.HiveServerContainer=DEBUG``` in log4j.properties. ## [3.1.0] - 2016-10-17 ### Added - Added methods to the shell that allow statements contained in files to be executed and their results gathered. These are particularly useful for HQL scripts that generate no table based data and instead write results to STDOUT. In practice we've seen these scripts used in data processing job orchestration scripts (e.g `bash`) to check for new data, calculate processing boundaries, etc. These values are then used to appropriately configure and launch some downstream job. - Support abstract base class [#48](https://github.com/klarna/HiveRunner/issues/48). ## [3.0.0] - 2016-02-05 ### Changed - Upgraded to Hive 1.2.1 (Note: new major release with backwards incompatibility issues). As of Hive 1.2 there are a number of new reserved keywords, see [DDL manual](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-Keywords,Non-reservedKeywordsandReservedKeywords) for more information. If you happen to have one of these as an identifier, you could either backtick quote them (e.g. \`date\`, \`timestamp\` or \`update\`) or set hive.support.sql11.reserved.keywords=false. - Users of Hive version 0.14 or older are recommended to use HiveRunner version 2.6.0. - Removed the custom HiveConf hive.vs. Use hadoop.tmp.dir instead. ## [2.6.0] - 2015-12-01 ### Added - Introduced command shell emulations to replicate different handling of full line comments in `hive` and `beeline` shells. Now strips full line comments for executed scripts to match the behaviour of the `hive -f` file option. - Option to use files as input for com.klarna.hiverunner.HiveShell.execute(...). ## [2.5.1] - 2015-11-12 ### Changed - Fixed deadlock in `ThrowOnTimeout.java` that occurred when running with long running test case and disabled timeout. ## [2.5.0] ### Added - Added support with `HiveShell.insertInto` for fluently generating test data in a table storage format agnostic manner. ## [2.4.0] ### Changed - Enabled any hiveconf variables to be set as System properties by using the naming convention hiveconf_[HiveConf property name]. e.g: hiveconf_hive.execution.engine. - Fixed bug: Results sets bigger than 100 rows only returned the first 100 rows. ## [2.3.0] ### Changed - Merged Tez and MR context into the same context again. Now, the same test suite may alter between execution engines by doing e.g.: hive> set hive.execution.engine=tez; hive> [some query] hive> set hive.execution.engine=mr; hive> [some query] ## [2.2.0] ### Added - Added support for setting hivevars via HiveShell. ================================================ FILE: CODE-OF-CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behaviour that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behaviour by participants include: * The use of sexualised language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behaviour and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behaviour. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviours that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported by contacting [a member of the project team](https://github.com/orgs/HiveRunner/teams/hiverunner-committers). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html [homepage]: https://www.contributor-covenant.org ================================================ FILE: CONTRIBUTING.md ================================================ # How To Contribute We'd love to accept your patches and contributions to this project. There are just a few guidelines you need to follow which are described in detail below. ## 1. Fork this repo You should create a fork of this project in your account and work from there. You can create a fork by clicking the fork button in GitHub. ## 2. One feature, one branch Work for each new feature/issue should occur in its own branch. To create a new branch from the command line: ```shell git checkout -b my-new-feature ``` where "my-new-feature" describes what you're working on. ## 3. Add unit tests If your contribution modifies existing or adds new code please add corresponding unit tests for this. ## 4. Ensure that the build passes Run ```shell mvn package ``` and check that there are no errors. ## 5. Add documentation for new or updated functionality Please review all of the .md files in this project to see if they are impacted by your change and update them accordingly. ## 6. Add to CHANGELOG.md Any notable changes should be recorded in the CHANGELOG.md following the [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) conventions. ## 7. Submit a pull request and describe the change Push your changes to your branch and open a pull request against the parent repo on GitHub. The project administrators will review your pull request and respond with feedback. # How your contribution gets merged Upon pull request submission, your code will be reviewed by the maintainers. They will confirm at least the following: - Tests run successfully (unit, coverage, integration, style). - Contribution policy has been followed. Two (human) reviewers will need to sign off on your pull request before it can be merged. ================================================ FILE: LICENSE.txt ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ [![Maven Central](https://maven-badges.herokuapp.com/maven-central/io.github.hiverunner/hiverunner/badge.svg?subject=io.github.hiverunner:hiverunner)](https://maven-badges.herokuapp.com/maven-central/io.github.hiverunner/hiverunner) [![Build](https://github.com/HiveRunner/hiverunner/workflows/build/badge.svg)](https://github.com/HiveRunner/HiveRunner/actions?query=workflow:"build") [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![ScreenShot](/images/HiveRunnerSplash.png) # HiveRunner Welcome to HiveRunner - Zero installation open source unit testing of [Hive](https://hive.apache.org/) applications. [Watch the HiveRunner teaser on youtube!](http://youtu.be/B7yEAHwgi2w) Welcome to the open source project HiveRunner. HiveRunner is a unit test framework based on JUnit (4 & 5) and enables TDD development of Hive SQL without the need for any installed dependencies. All you need is to add HiveRunner to your `pom.xml` as any other library and you're good to go. HiveRunner is under constant development. It is used extensively by many companies. Please feel free to suggest improvements both as pull requests and as written requests. ## Overview HiveRunner enables you to write Hive SQL as releasable tested artifacts. It will require you to parametrize and modularize Hive SQL in order to make it testable. The bits and pieces of code should then be wired together with some orchestration/workflow/build tool of your choice, to be runnable in your environment (e.g. Oozie, Pentaho, Talend, Maven, etc.) So, even though your current Hive SQL probably won't run off the shelf within HiveRunner, we believe the enforced testability and enabling of a TDD workflow will do as much good to the scripting world of SQL as it has for the Java community. ## Versions Different versions of HiveRunner target different versions of Hive as follows: | HiveRunner Version | Hive Version | Status | Source Code Branch | |--------------------|--------------|----------------------------|--------------------------------------------------------| | 7.x | 4.x | New, active development | https://github.com/HiveRunner/HiveRunner/tree/hive-4.x | | 6.x | 3.x | Stable, active development | https://github.com/HiveRunner/HiveRunner (i.e. `main`) | | 5.x | 2.x | Stable, bug fixes only | https://github.com/HiveRunner/HiveRunner/tree/hive-2.x | # Cook Book ## 1. Include HiveRunner HiveRunner is published to [Maven Central](https://search.maven.org/search?q=hiverunner). To start to use it, add a dependency to HiveRunner to your pom file: io.github.hiverunner hiverunner [HIVERUNNER VERSION] test Alternatively, if you want to build from source, clone this repo and build with: mvn install Then add the dependency as mentioned above. Also explicitly add the surefire plugin and configure forkMode=always to avoid OutOfMemory when building big test suites. org.apache.maven.plugins maven-surefire-plugin 2.21.0 always As an alternative if this does not solve the OOM issues, try increase the -Xmx and -XX:MaxPermSize settings. For example: org.apache.maven.plugins maven-surefire-plugin 2.21.0 1 false -Xmx2048m -XX:MaxPermSize=512m (please note that the forkMode option is deprecated and you should use forkCount and reuseForks instead) With forkCount and reuseForks there is a possibility to reduce the test execution time drastically, depending on your hardware. A plugin configuration which are using one fork per CPU core and reuse threads would look like: org.apache.maven.plugins maven-surefire-plugin 2.21.0 1C true -Xmx2048m -XX:MaxPermSize=512m By default, HiveRunner uses mapreduce (mr) as the execution engine for Hive. If you wish to run using Tez, set the System property `hiveconf_hive.execution.engine` to 'tez'. (Any Hive conf property may be overridden by prefixing it with 'hiveconf_') org.apache.maven.plugins maven-surefire-plugin 2.21.0 tez 1000 ### Timeout It's possible to configure HiveRunner to make tests time out after some time and retry those tests a couple of times, but only when using `StandaloneHiveRunner` as this is not available in the `HiveRunnerExtension` (from HiveRunner 5.x and up). This is to cover for the bug https://issues.apache.org/jira/browse/TEZ-2475 that at times causes test cases to not terminate due to a lost DAG reference. The timeout feature can be configured via the 'enableTimeout', 'timeoutSeconds' and 'timeoutRetries' properties. A configuration which enables timeouts after 30 seconds and allows 2 retries would look like: org.apache.maven.plugins maven-surefire-plugin 2.21.0 true 30 2 ### Logging HiveRunner uses [SLF4J](https://www.slf4j.org/) so you should configure logging in your tests using any compatible logging framework. ## 2. Look at the examples Look at the [com.klarna.hiverunner.examples.HelloHiveRunnerTest](/src/test/java/com/klarna/hiverunner/examples/HelloHiveRunnerTest.java) reference test case to get a feeling for how a typical test case looks like in JUnit5. To find JUnit4 versions of the examples, look at [com.klarna.hiverunner.examples.junit4.HelloHiveRunnerTest](/src/test/java/com/klarna/hiverunner/examples/junit4/HelloHiveRunnerTest.java). If you're put off by the verbosity of the annotations, there's always the possibility to use HiveShell in a more interactive mode. The [com.klarna.hiverunner.SerdeTest](/src/test/java/com/klarna/hiverunner/SerdeTest.java) adds a resource (test data) interactively with HiveShell instead of using annotations. Annotations and interactive mode can be mixed and matched, however you'll always need to include the [com.klarna.hiverunner.annotations.HiveSQL](/src/main/java/com/klarna/hiverunner/annotations/HiveSQL.java) annotation e.g: @HiveSQL(files = {"serdeTest/create_table.sql", "serdeTest/hql_custom_serde.sql"}, autoStart = false) public HiveShell hiveShell; Note that the *autostart = false* is needed for the interactive mode. It can be left out when running with only annotations. ### Sequence files If you work with __sequence files__ (Or anything else than regular text files) make sure to take a look at [ResourceOutputStreamTest](/src/test/java/com/klarna/hiverunner/ResourceOutputStreamTest.java) for an example of how to use the new method [HiveShell](src/main/java/com/klarna/hiverunner/HiveShell.java)\#getResourceOutputStream to manage test input data. ### Programatically create test input data Test data can be programmatically inserted into any Hive table using `HiveShell.insertInto(...)`. This seamlessly handles different storage formats and partitioning types allowing you to focus on the data required by your test scenarios: hiveShell.execute("create database test_db"); hiveShell.execute("create table test_db.test_table (" + "c1 string," + "c2 string," + "c3 string" + ")" + "partitioned by (p1 string)" + "stored as orc"); hiveShell.insertInto("test_db", "test_table") .withColumns("c1", "p1").addRow("v1", "p1") // add { "v1", null, null, "p1" } .withAllColumns().addRow("v1", "v2", "v3", "p1") // add { "v1", "v2", "v3", "p1" } .copyRow().set("c1", "v4") // add { "v4", "v2", "v3", "p1" } .addRowsFromTsv(file) // parses TSV data out of a file resource .addRowsFrom(file, fileParser) // parses custom data out of a file resource .commit(); See [com.klarna.hiverunner.examples.InsertTestDataTest](/src/test/java/com/klarna/hiverunner/examples/InsertTestDataTest.java) for working examples. ## 3. Understand the order of execution HiveRunner will in default mode set up and start the HiveShell before the test method is invoked. If autostart is set to false, the [HiveShell](/src/main/java/com/klarna/hiverunner/HiveShell.java) must be started manually from within the test method. Either way, HiveRunner will do the following steps when start is invoked: 1. Merge any [@HiveProperties](/src/main/java/com/klarna/hiverunner/annotations/HiveProperties.java) from the test case with the Hive conf 2. Start the HiveServer with the merged conf 3. Copy all [@HiveResource](/src/main/java/com/klarna/hiverunner/annotations/HiveResource.java) data into the temp file area for the test 4. Execute all fields annotated with [@HiveSetupScript](/src/main/java/com/klarna/hiverunner/annotations/HiveSetupScript.java) 5. Execute the script files given in the [@HiveSQL](/src/main/java/com/klarna/hiverunner/annotations/HiveSQL.java) annotation The [HiveShell](/src/main/java/com/klarna/hiverunner/HiveShell.java) field annotated with [@HiveSQL](/src/main/java/com/klarna/hiverunner/annotations/HiveSQL.java) will always be injected before the test method is invoked. # Hive version compatibility - This version of HiveRunner is built for Hive 3.1.2. - For Hive 2.x support please use HiveRunner 5.x. - Command shell emulations are provided to closely match the behaviour of both the Hive CLI and Beeline interactive shells. The desired emulation can be specified in your `pom.xml` file like so: org.apache.maven.plugins maven-surefire-plugin 2.21.0 BEELINE Or provided on the command line using a system property: mvn -DcommandShellEmulator=BEELINE test # Future work and Limitations * HiveRunner does not allow the `add jar` statement. It is considered bad practice to keep environment specific code together with the business logic that targets HiveRunner. Keep environment specific stuff in separate files and use your build/orchestration/workflow tool to run the right files in the right order in the right environment. When running HiveRunner, all SerDes available on the classpath of the IDE/maven will be available. * HiveRunner runs Hive and Hive runs on top of Hadoop, and Hadoop has limited support for Windows machines. Installing [Cygwin](http://www.cygwin.com/ "Cygwin") might help out. * Currently the HiveServer spins up and tears down for every test method. As a performance option it should be possible to clean the HiveServer and metastore between each test method invocation. The choice should probably be exposed to the test writer. By switching between different strategies, side effects/leakage can be ruled out during test case debugging. See [#69](https://github.com/HiveRunner/HiveRunner/issues/69). # Known Issues ### UnknownHostException I've had issues with UnknownHostException on OS X after upgrading my system or running Docker. Usually a restart of my machine solved it, but last time I got some corporate stuff installed the restarts stopped working and I kept getting UnknownHostExceptions. Following this simple guide solved my problem: http://crunchify.com/getting-java-net-unknownhostexception-nodename-nor-servname-provided-or-not-known-error-on-mac-os-x-update-your-privateetchosts-file/ ### Tez queries do not terminate Tez will at times forget the process id of a random DAG. This will cause the query to never terminate. To get around this there is a timeout and retry functionality implemented in HiveRunner: org.apache.maven.plugins maven-surefire-plugin 2.21.0 true 30 2 Make sure to set the timeoutSeconds to that of your slowest test in the test suite and then add some padding. # Contact # Mailing List If you would like to ask any questions about or discuss HiveRunner please join our mailing list at [https://groups.google.com/forum/#!forum/hive-runner-user](https://groups.google.com/forum/#!forum/hive-runner-user) # History This project was initially developed and maintained by [Klarna](https://klarna.github.io/) and then by [Expedia Group](https://expediagroup.github.io/) before moving to its own top-level organisation on GitHub. # Legal This project is available under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0.html). Copyright 2021-2024 The HiveRunner Contributors. Copyright 2013-2021 Klarna AB. ================================================ FILE: RELEASING.md ================================================ # Releasing HiveRunner to Maven Central HiveRunner has been set up to build continuously and also to deploy SNAPSHOTS to Sonatype and releases to Maven Central via GitHub Actions. ## Deploying a SNAPSHOT to Sonatype * Select the https://github.com/HiveRunner/HiveRunner/actions/workflows/deploy.yml worfklow * Click "Run workflow" * Select the branch to use to deploy a SNAPSHOT from: * Use `main` as the branch for a Hive 3.x release * Use `hive-4.x` as the branch for a Hive 4.x release * Use `hive-2.x` as the branch for a Hive 2.x release * Run the workflow * SNAPSHOT artifacts will be available at https://s01.oss.sonatype.org/content/repositories/snapshots/io/github/hiverunner/hiverunner/ ## Deploying a release to Maven Central * Ensure the `pom.xml` has the SNAPSHOT version set to the value you would like to make a release from * Update `CHANGELOG.md` with the date corresponding to when you're performing the release * Select the https://github.com/HiveRunner/HiveRunner/actions/workflows/release.yml worfklow * Click "Run workflow" * Select the branch to use to deploy a release from: * Use `main` as the branch for a Hive 3.x release * Use `hive-4.x` as the branch for a Hive 4.x release * Use `hive-2.x` as the branch for a Hive 2.x release * Run the workflow * Release artifacts will be available at https://repo1.maven.org/maven2/io/github/hiverunner/hiverunner/ * It can take a few hours before the artifacts show up in searches performed at https://search.maven.org/search?q=hiverunner ================================================ FILE: pom.xml ================================================ 4.0.0 io.github.hiverunner hiverunner 6.1.1-SNAPSHOT HiveRunner HiveRunner is a unit test framework based on JUnit (4 or 5) that enables TDD development of Hive SQL without the need of any installed dependencies. https://github.com/HiveRunner/HiveRunner 2021 The Apache Software License, Version 2.0 http://www.apache.org/licenses/LICENSE-2.0.txt repo HiveRunner Committers HiveRunner https://github.com/HiveRunner scm:git:https://github.com/HiveRunner/HiveRunner.git scm:git:https://github.com/HiveRunner/HiveRunner.git git@github.com:HiveRunner/HiveRunner.git HEAD UTF-8 3.1.0 mr 3.1.2 5.7.1 3.0 0.9.1 3.8.0 org.apache.hive hive-serde ${hive.version} com.fasterxml.jackson.core jackson-annotations 2.9.5 org.apache.hive hive-jdbc ${hive.version} org.apache.hive.hcatalog hive-webhcat-java-client ${hive.version} org.apache.hive hive-service ${hive.version} tez-dag org.apache.tez ${tez.version} tez-common org.apache.tez ${tez.version} tez-mapreduce org.apache.tez ${tez.version} org.apache.hive hive-contrib ${hive.version} test org.reflections reflections 0.9.8 org.apache.hadoop hadoop-mapreduce-client-common ${hadoop.version} org.apache.hadoop hadoop-mapreduce-client-core ${hadoop.version} com.github.stefanbirkner system-rules 1.19.0 test org.hamcrest hamcrest-all 1.3 test org.mockito mockito-core ${mockito.version} test org.mockito mockito-junit-jupiter ${mockito.version} test junit junit 4.13.2 provided org.junit.jupiter junit-jupiter ${junit.jupiter.version} org.junit.vintage junit-vintage-engine ${junit.jupiter.version} org.apache.maven.plugins maven-surefire-plugin 2.22.2 -Xmx2024m false ${hive.execution.engine} 1000 false 30 2 org.apache.maven.plugins maven-compiler-plugin 3.7.0 1.8 1.8 org.apache.maven.plugins maven-jar-plugin 3.2.0 ${project.version} ${maven.build.timestamp} ${project.groupId} ${project.artifactId} com.mycila license-maven-plugin ${license.maven.plugin.version} com.mycila license-maven-plugin-git ${license.maven.plugin.version}
src/main/license/APACHE-2.txt
The HiveRunner Contributors src/main/java/** src/test/java/** validate format org.apache.maven.plugins maven-release-plugin 3.0.0-M1 v@{project.version} true false oss-release deploy org.sonatype.plugins nexus-staging-maven-plugin 1.6.8 true ossrh https://s01.oss.sonatype.org/ true oss-release ossrh https://s01.oss.sonatype.org/content/repositories/snapshots ossrh https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ org.apache.maven.plugins maven-source-plugin 3.2.0 attach-sources jar-no-fork org.apache.maven.plugins maven-javadoc-plugin 3.2.0 ${project.build.sourceEncoding} 8 false none attach-javadocs jar org.apache.maven.plugins maven-gpg-plugin 1.6 sign-artifacts verify sign --pinentry-mode loopback tez tez ================================================ FILE: src/main/java/com/klarna/hiverunner/HiveRunnerCore.java ================================================ /** * Copyright (C) 2013-2021 Klarna AB * Copyright (C) 2021 The HiveRunner Contributors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.klarna.hiverunner; import static org.reflections.ReflectionUtils.withAnnotation; import java.io.File; import java.io.IOException; import java.lang.reflect.Field; import java.net.URISyntaxException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; import com.google.common.base.Preconditions; import com.google.common.io.Resources; import com.klarna.hiverunner.annotations.HiveProperties; import com.klarna.hiverunner.annotations.HiveResource; import com.klarna.hiverunner.annotations.HiveSQL; import com.klarna.hiverunner.annotations.HiveSetupScript; import com.klarna.hiverunner.builder.HiveShellBuilder; import com.klarna.hiverunner.builder.Script; import com.klarna.hiverunner.config.HiveRunnerConfig; import com.klarna.reflection.ReflectionUtils; class HiveRunnerCore { /** * Traverses the test case annotations. Will inject a HiveShell in the test case that envelopes the HiveServer. */ HiveShellContainer createHiveServerContainer(List scripts, Object testCase, Path baseDir, HiveRunnerConfig config) throws IOException { HiveServerContext context = new StandaloneHiveServerContext(baseDir, config); return buildShell(scripts, testCase, config, context); } private HiveShellContainer buildShell(List scripts, Object testCase, HiveRunnerConfig config, HiveServerContext context) throws IOException { HiveServerContainer hiveTestHarness = new HiveServerContainer(context); HiveShellBuilder hiveShellBuilder = new HiveShellBuilder(); hiveShellBuilder.setCommandShellEmulation(config.getCommandShellEmulator()); HiveShellField shellSetter = loadScriptUnderTest(testCase, hiveShellBuilder); if (!scripts.isEmpty()) { hiveShellBuilder.overrideScriptsUnderTest(scripts); } hiveShellBuilder.setHiveServerContainer(hiveTestHarness); loadAnnotatedResources(testCase, hiveShellBuilder); loadAnnotatedProperties(testCase, hiveShellBuilder); loadAnnotatedSetupScripts(testCase, hiveShellBuilder); // Build shell HiveShellContainer shell = hiveShellBuilder.buildShell(); // Set shell shellSetter.setShell(shell); if (shellSetter.isAutoStart()) { shell.start(); } return shell; } private HiveShellField loadScriptUnderTest(Object testCaseInstance, HiveShellBuilder hiveShellBuilder) { try { Set fields = ReflectionUtils.getAllFields(testCaseInstance.getClass(), withAnnotation(HiveSQL.class)); Preconditions.checkState(fields.size() == 1, "Exact one field should to be annotated with @HiveSQL"); Field field = fields.iterator().next(); HiveSQL annotation = field.getAnnotation(HiveSQL.class); List scriptPaths = getScriptPaths(annotation); Charset charset = annotation.encoding().equals("") ? Charset.defaultCharset() : Charset.forName(annotation.encoding()); boolean isAutoStart = annotation.autoStart(); hiveShellBuilder.setScriptsUnderTest(scriptPaths, charset); return new HiveShellField() { @Override public void setShell(HiveShell shell) { ReflectionUtils.setField(testCaseInstance, field.getName(), shell); } @Override public boolean isAutoStart() { return isAutoStart; } }; } catch (Throwable t) { throw new IllegalArgumentException("Failed to init field annotated with @HiveSQL: " + t.getMessage(), t); } } protected List getScriptPaths(HiveSQL annotation) throws URISyntaxException { List scriptPaths = new ArrayList<>(); for (String scriptFilePath : annotation.files()) { Path file = Paths.get(Resources.getResource(scriptFilePath).toURI()); assertFileExists(file); scriptPaths.add(file); } return scriptPaths; } private void assertFileExists(Path file) { Preconditions.checkState(Files.exists(file), "File " + file + " does not exist"); } private void loadAnnotatedSetupScripts(Object testCase, HiveShellBuilder workFlowBuilder) { Set setupScriptFields = ReflectionUtils.getAllFields(testCase.getClass(), withAnnotation(HiveSetupScript.class)); for (Field setupScriptField : setupScriptFields) { if (ReflectionUtils.isOfType(setupScriptField, String.class)) { String script = ReflectionUtils.getFieldValue(testCase, setupScriptField.getName(), String.class); workFlowBuilder.addSetupScript(script); } else if (ReflectionUtils.isOfType(setupScriptField, File.class) || ReflectionUtils.isOfType(setupScriptField, Path.class)) { Path path = getMandatoryPathFromField(testCase, setupScriptField); workFlowBuilder.addSetupScript(readAll(path)); } else { throw new IllegalArgumentException( "Field annotated with @HiveSetupScript currently only supports type String, File and Path"); } } } private static String readAll(Path path) { try { return new String(Files.readAllBytes(path), StandardCharsets.UTF_8); } catch (IOException e) { throw new IllegalStateException("Unable to read " + path + ": " + e.getMessage(), e); } } private void loadAnnotatedResources(Object testCase, HiveShellBuilder workFlowBuilder) throws IOException { Set fields = ReflectionUtils.getAllFields(testCase.getClass(), withAnnotation(HiveResource.class)); for (Field resourceField : fields) { HiveResource annotation = resourceField.getAnnotation(HiveResource.class); String targetFile = annotation.targetFile(); if (ReflectionUtils.isOfType(resourceField, String.class)) { String data = ReflectionUtils.getFieldValue(testCase, resourceField.getName(), String.class); workFlowBuilder.addResource(targetFile, data); } else if (ReflectionUtils.isOfType(resourceField, File.class) || ReflectionUtils.isOfType(resourceField, Path.class)) { Path dataFile = getMandatoryPathFromField(testCase, resourceField); workFlowBuilder.addResource(targetFile, dataFile); } else { throw new IllegalArgumentException( "Fields annotated with @HiveResource currently only supports field type String, File or Path"); } } } private Path getMandatoryPathFromField(Object testCase, Field resourceField) { Path path; if (ReflectionUtils.isOfType(resourceField, File.class)) { File dataFile = ReflectionUtils.getFieldValue(testCase, resourceField.getName(), File.class); path = Paths.get(dataFile.toURI()); } else if (ReflectionUtils.isOfType(resourceField, Path.class)) { path = ReflectionUtils.getFieldValue(testCase, resourceField.getName(), Path.class); } else { throw new IllegalArgumentException( "Only Path or File type is allowed on annotated field " + resourceField); } Preconditions.checkArgument(Files.exists(path), "File %s does not exist", path); return path; } private void loadAnnotatedProperties(Object testCase, HiveShellBuilder workFlowBuilder) { for (Field hivePropertyField : ReflectionUtils.getAllFields(testCase.getClass(), withAnnotation(HiveProperties.class))) { Preconditions.checkState(ReflectionUtils.isOfType(hivePropertyField, Map.class), "Field annotated with @HiveProperties should be of type Map"); workFlowBuilder.putAllProperties( ReflectionUtils.getFieldValue(testCase, hivePropertyField.getName(), Map.class)); } } /** * Used as a handle for the HiveShell field in the test case so that we may set it once the * HiveShell has been instantiated. */ interface HiveShellField { void setShell(HiveShell shell); boolean isAutoStart(); } } ================================================ FILE: src/main/java/com/klarna/hiverunner/HiveRunnerExtension.java ================================================ /** * Copyright (C) 2013-2021 Klarna AB * Copyright (C) 2021 The HiveRunner Contributors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.klarna.hiverunner; import static org.reflections.ReflectionUtils.withAnnotation; import static org.reflections.ReflectionUtils.withType; import java.io.IOException; import java.io.UncheckedIOException; import java.lang.reflect.Field; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Set; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.TestInstancePostProcessor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; import com.google.common.base.Predicates; import com.klarna.hiverunner.annotations.HiveRunnerSetup; import com.klarna.hiverunner.annotations.HiveSQL; import com.klarna.hiverunner.builder.Script; import com.klarna.hiverunner.config.HiveRunnerConfig; import com.klarna.reflection.ReflectionUtils; public class HiveRunnerExtension implements AfterEachCallback, TestInstancePostProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(HiveRunnerExtension.class); private final HiveRunnerCore core; private final HiveRunnerConfig config = new HiveRunnerConfig(); private Path basedir; private HiveShellContainer container; protected List