Showing preview only (438K chars total). Download the full file or copy to clipboard to get everything.
Repository: walmartlabs/bigben
Branch: master
Commit: 61f4760a6617
Files: 99
Total size: 405.4 KB
Directory structure:
gitextract_bc7hdl64/
├── .gitignore
├── .looper.yml
├── LICENSE.txt
├── README.md
├── app/
│ ├── LICENSE.txt
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── kotlin/
│ │ │ └── com/
│ │ │ └── walmartlabs/
│ │ │ └── bigben/
│ │ │ └── app/
│ │ │ ├── app.kt
│ │ │ └── run.kt
│ │ └── resources/
│ │ ├── application.conf
│ │ ├── bigben-lifecycle.yaml
│ │ ├── bigben.yaml
│ │ └── log4j.xml
│ └── test/
│ ├── kotlin/
│ │ └── com/
│ │ └── walmartlabs/
│ │ └── bigben/
│ │ └── tests/
│ │ ├── APITests.kt
│ │ ├── BigBenTests.kt
│ │ └── KafkaTests.kt
│ └── resources/
│ ├── bigben-api-test.yaml
│ ├── bigben-kafka-test.yaml
│ ├── bigben-test.yaml
│ └── log4j.xml
├── build/
│ ├── configs/
│ │ ├── log4j.xml
│ │ └── overrides.yaml
│ ├── docker/
│ │ ├── Dockerfile
│ │ ├── app_run.sh
│ │ ├── cassandra_run.sh
│ │ ├── cleanup.sh
│ │ ├── deploy.sh
│ │ ├── docker-compose.yml
│ │ ├── docker_build.sh
│ │ ├── single_node_run.sh
│ │ └── start.sh
│ └── exec/
│ ├── app_run.sh
│ ├── build.sh
│ └── cleanup.sh
├── cassandra/
│ ├── LICENSE.txt
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── kotlin/
│ │ │ └── com/
│ │ │ └── walmartlabs/
│ │ │ └── bigben/
│ │ │ └── providers/
│ │ │ └── domain/
│ │ │ └── cassandra/
│ │ │ ├── CassandraModule.kt
│ │ │ ├── ClusterConfig.kt
│ │ │ ├── Entities.kt
│ │ │ └── codecs.kt
│ │ └── resources/
│ │ └── bigben-schema.cql
│ └── test/
│ ├── kotlin/
│ │ └── com/
│ │ └── walmartlabs/
│ │ └── bigben/
│ │ └── cassandra/
│ │ └── tests/
│ │ ├── IntegrationTests.kt
│ │ └── ORMTests.kt
│ └── resources/
│ ├── bigben-test.yaml
│ ├── log4j.xml
│ └── testng.xml
├── commons/
│ ├── LICENSE.txt
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ └── kotlin/
│ │ └── com/
│ │ └── walmartlabs/
│ │ └── bigben/
│ │ └── utils/
│ │ ├── _extns.kt
│ │ ├── _future_extns.kt
│ │ ├── commons/
│ │ │ ├── ListenableFutureAdapter.kt
│ │ │ ├── Props.kt
│ │ │ ├── TaskExecutor.kt
│ │ │ └── modules.kt
│ │ └── hz/
│ │ ├── ClusterSingleton.kt
│ │ ├── Hz.kt
│ │ └── Service.kt
│ └── test/
│ ├── kotlin/
│ │ └── PropsTests.kt
│ └── resources/
│ ├── a.yaml
│ ├── b.yaml
│ ├── log4j.xml
│ ├── overrides.yaml
│ ├── props.yaml
│ ├── sub1-overrides.yaml
│ └── sub1.yaml
├── cron/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── kotlin/
│ └── com/
│ └── walmartlabs/
│ └── bigben/
│ └── cron/
│ ├── cron-hz.kt
│ ├── cron-processors.kt
│ └── cron.kt
├── kafka/
│ ├── LICENSE.txt
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── kotlin/
│ └── com/
│ └── walmartlabs/
│ └── bigben/
│ └── kafka/
│ ├── kafka-mocks.kt
│ ├── kafka-module.kt
│ ├── kafka-processor.kt
│ └── kafka-producer.kt
├── lib/
│ ├── LICENSE.txt
│ ├── pom.xml
│ └── src/
│ └── main/
│ ├── kotlin/
│ │ └── com/
│ │ └── walmartlabs/
│ │ └── bigben/
│ │ ├── BigBen.kt
│ │ ├── api/
│ │ │ ├── EventReceiver.kt
│ │ │ └── EventService.kt
│ │ ├── core/
│ │ │ ├── BucketManager.kt
│ │ │ ├── BucketSnapshot.kt
│ │ │ ├── BucketsLoader.kt
│ │ │ ├── ScheduleScanner.kt
│ │ │ └── StatusSyncer.kt
│ │ ├── entities/
│ │ │ ├── EntityProvider.kt
│ │ │ └── entities.kt
│ │ ├── extns/
│ │ │ ├── _api_response_extns.kt
│ │ │ ├── _bigben_extns.kt
│ │ │ ├── _do_extns.kt
│ │ │ └── _time_extns.kt
│ │ ├── hz/
│ │ │ ├── BucketStore.kt
│ │ │ └── HzObjectFactory.kt
│ │ ├── modules.kt
│ │ ├── processors/
│ │ │ ├── no_ops.kt
│ │ │ └── processors.kt
│ │ └── tasks/
│ │ └── tasks.kt
│ └── resources/
│ └── hz.template.xml
├── pom.xml
└── run_bigben_standalone.sh
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
**/target/
# Mobile Tools for Java (J2ME)
.mtj.tmp/
# Package Files #
*.ear
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
pom.xml.tag
pom.xml.releaseBackup
pom.xml.versionsBackup
pom.xml.next
release.properties
dependency-reduced-pom.xml
buildNumber.properties
.DS_Store
.idea/
.modules/
app/app.iml
cassandra/bigben-cassandra.iml
commons/bigben-commons.iml
cron/cron.iml
kafka/bigben-kafka.iml
lib/bigben-lib.iml
================================================
FILE: .looper.yml
================================================
tools:
jdk: 8
maven: 3.5.2
triggers:
- manual: Run default
- manual:
name: Release Build
call: release
flows:
default:
- call: versionsCheck
- call: build
pr:
- echo "Running build for $GITHUB_PR_URL"
- call: versionsCheck
- (name Maven build) mvn -B clean install
versionsCheck:
- (name JDK Version) java -version
- (name Maven version) mvn -v
build:
- exposeVars(maven)
- (name Project information) echo "Building ${MAVEN_GROUP_ID}:${MAVEN_ARTIFACT_ID}:${MAVEN_VERSION}"
- (name Maven deploy) mvn -B -DskipTests -Darguments=-DskipTests clean deploy
release:
- call: versionsCheck
- (name Maven release) mvn -B -DskipTests -Darguments=-DskipTests clean release:prepare release:perform
================================================
FILE: LICENSE.txt
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2018 Sandeep Malik
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
***
# NOTICE:
## This repository has been archived and is not supported.
[](http://unmaintained.tech/)
***
NOTICE: SUPPORT FOR THIS PROJECT HAS ENDED
This projected was owned and maintained by Walmart. This project has reached its end of life and Walmart no longer supports this project.
We will no longer be monitoring the issues for this project or reviewing pull requests. You are free to continue using this project under the license terms or forks of this project at your own risk. This project is no longer subject to Walmart's bug bounty program or other security monitoring.
## Actions you can take
We recommend you take the following action:
* Review any configuration files used for build automation and make appropriate updates to remove or replace this project
* Notify other members of your team and/or organization of this change
* Notify your security team to help you evaluate alternative options
## Forking and transition of ownership
For [security reasons](https://www.theregister.co.uk/2018/11/26/npm_repo_bitcoin_stealer/), Walmart does not transfer the ownership of our primary repos on Github or other platforms to other individuals/organizations. Further, we do not transfer ownership of packages for public package management systems.
If you would like to fork this package and continue development, you should choose a new name for the project and create your own packages, build automation, etc.
Please review the licensing terms of this project, which continue to be in effect even after decommission.
# BigBen
`BigBen` is a generic, multi-tenant, time-based event scheduler and cron
scheduling framework based on `Cassandra` and `Hazelcast`
It has following features:
* **Distributed** - `BigBen` uses a distributed design and can be deployed on 10's or 100's of machines and can be dc-local or cross-dc
* **Horizontally scalable** - `BigBen` scales linearly with the number of machines.
* **Fault tolerant** - `BigBen` employs a number of failure protection modes and
can withstand arbitrary prolonged down times
* **Performant** - `BigBen` can easily scale to 10,000's or even millions's of event triggers with a
very small cluster of machines. It can also easily manage million's of crons running in a distributed manner
* **Highly Available** - As long as a single machine is available in the cluster, `BigBen` will guarantee the
execution of events (albeit with a lower throughput)
* **Extremely consistent** - `BigBen` employs a single master design (the master itself is highly available with
`n-1` masters on standby in an `n` cluster machine) to ensure that no two nodes fire the same event or execute
the same cron.
* **NoSql based** - `BigBen` comes with default implementation with `Cassandra` but can be easily extended
to support other `NoSql` or even `RDBMS` data stores
* **Auditable** - `BigBen` keeps a track of all the events fired and crons executed with a configurable
retention
* **Portable, cloud friendly** - `BigBen` comes as application bundled as `war` or an embedded lib as `jar`,
and can be deployed on any cloud, `on-prem` or `public`
## Use cases
`BigBen` can be used for a variety of time based workloads, both single trigger based or repeating crons.
Some of the use cases can be
* **Delayed execution** - E.g. if a job is to be executed 30 mins from now
* **System retries** - E.g. if a service A wants to call service B and service B is down at the moment, then
service A can schedule an exponential backoff retry strategy with retry intervals of
1 min, 10 mins, 1 hour, 12 hours, and so on.
* **Timeout tickers** - E.g. if service A sends a message to service B via `Kafka` and expects a response in 1 min,
then it can schedule a `timeout check` event to be executed after 1 min
* **Polling services** - E.g. if service A wants to poll service B at some frequency, it can schedule a cron
to be executed at some specified frequency
* **Notification Engine** - `BigBen` can be used to implement `notification engine` with scheduled deliveries,
scheduled polls, etc
* **Workflow state machine** - `BigBen` can be used to implement a distributed `workflow` with state suspensions,
alerts and monitoring of those suspensions.
## Architectural Goals
`BigBen` was designed to achieve the following goals:
* Uniformly distributed storage model
* Resilient to hot spotting due to sudden surge in traffic
* Uniform execution load profile in the cluster
* Ensure that all nodes have similar load profiles to minimize misfires
* Linear Horizontal Scaling
* Lock-free execution
* Avoid resource contentions
* Plugin based architecture to support variety of data bases like `Cassandra, Couchbase, Solr Cloud, Redis, RDBMS`, etc
* Low maintenance, elastic scaling
## Design and architecture
See the blog published at [Medium](https://medium.com/walmartlabs/an-approach-to-designing-distributed-fault-tolerant-horizontally-scalable-event-scheduler-278c9c380637)
for a full description of various design elements of `BigBen`
## Events Inflow
`BigBen` can receive events in two modes:
* **kafka** - inbound and outbound Kafka topics to consume event requests and publish event triggers
* **http** - HTTP APIs to send event requests and HTTP APIs to receive event triggers.
*It is strongly recommended to use `kafka` for better scalability*
### Event Inflow diagram
*Request and Response channels can be mixed. For example, the event requests can be sent through HTTP APIs but
the event triggers (response) can be received through a Kafka Topic.*
## Event processing guarantees
`BigBen` has a robust event processing guarantees to survive various failures.
However, `event-processing` is not same as `event-acknowledgement`.
`BigBen` works in a no-acknowledgement mode (*at least for now*).
Once an event is triggered, it is either published to `Kafka` or
sent through an `HTTP API`. Once the `Kafka` producer returns success, or `HTTP API` returns non-500 status code,
the event is **assumed** to be processed and marked as such in the system.
However, for whatever reason if the event was not processed and resulted in an error
(e.g. `Kafka` producer timing out, or `HTTP API` throwing `503`),
then the event will be retried multiple times as per the strategies discussed below
### Event misfire strategy
Multiple scenarios can cause `BigBen` to be not able to trigger an event on time. Such scenarios are called
misfires. Some of them are:
* `BigBen`'s internal components are down during event trigger.
E.g.
* `BigBen`'s data store is down and events could not be fetched
* `VMs` are down
* `Kafka` Producer could not publish due to loss of partitions / brokers or any other reasons
* `HTTP API` returned a 500 error code
* Any other unexpected failure
In any of these cases, the event is first retried in memory using an exponential back-off strategy.
Following parameters control the retry behavior:
* _event.processor.max.retries_ - how many in-memory retries will be made before declaring the event as error, default is 3
* _event.processor.initial.delay_ - how long in seconds the system should wait before kicking in the retry, default is 1 second
* _event.processor.backoff.multiplier_ - the back off multiplier factor, default is 2. E.g. the intervals would be 1 second, 2 seconds, 4 seconds.
If the event still is not processed, then the event is marked as `ERROR`.
All the events marked `ERROR` are retried up to a configured limit called `events.backlog.check.limit`.
This value can be an arbitrary amount of time, e.g. 1 day, 1 week, or even 1 year. E.g. if the the limit
is set at `1 week` then any event failures will be retried for `1 week` after which, they will be permanently
marked as `ERROR` and ignored. The `events.backlog.check.limit` can be changed at any time by changing the
value in `bigben.yaml` file and bouncing the servers.
### Event bucketing and shard size
`BigBen` shards events by minutes. However, since it's not known in advance how many events will be
scheduled in a given minute, the buckets are further sharded by a pre defined shard size. The shard size is a
design choice that needs to be made before deployment. Currently, it's not possible to
change the shard size once defined.
An undersized shard value has minimal performance impact, however an oversized shard value may
keep some machines idling. The default value of `1000` is good enough for most practical purposes as long as
number of events to be scheduled per minute exceed `1000 x n`, where `n` is the number of machines in the cluster.
If the events to be scheduled are much less than `1000` then a smaller shard size may be chosen.
### Multi shard parallel processing
Each bucket with all its shards is distributed across the cluster for execution with an algorithm that ensures a
random and uniform distribution. The following diagram shows the execution flow.
### Multi-tenancy
Multiple tenants can use `BigBen` in parallel. Each one can configure how the events will be delivered once triggered.
Tenant 1 can configure the events to be delivered in `kafka` topic `t1`, where as tenant 2 can have them delivered
via a specific `http` url. The usage of tenants will become more clearer with the below explanation of `BigBen` APIs
## Docker support
BigBen is dockerized and image (`bigben`) is available on docker hub. The code also contains
scripts, which start `cassandra`, `hazelcast` and `app`.
To quickly set up the application for local dev testing, do the following steps:
1. `git clone $repo`
2. `cd bigben/build/docker`
3. execute `./docker_build.sh`
4. start cassandra container by executing `./cassandra_run.sh`
5. start app by executing `./app_run.sh`
6. To run multiple app nodes `export NUM_INSTANCES=3 && ./app_run.sh`
6. wait for application to start on port `8080`
7. verify that `curl http://localhost:8080/ping` returns `200`
8. Use `./cleanup.sh` to stop and remove all `BigBen` related containers
## Non-docker execution
`BigBen` can be run without docker as well. Following are the steps
1. `git clone $repo`
2. `cd bigben/build/exec`
3. execute `./build.sh`
4. execute `./app_run.sh`
## Env properties
You can set the following environment properties
1. `APP_CONTAINER_NAME` (default bigben_app)
2. `SERVER_PORT` (default 8080)
3. `HZ_PORT` (default 5701)
4. `NUM_INSTANCES` (default 1)
5. `LOGS_DIR` (default bigben/../bigben_logs)
6. `CASSANDRA_SEED_IPS` (default $HOST_IP)
7. `HZ_MEMBER_IPS` (default $HOST_IP)
8. `JAVA_OPTS`
#How to override default config values?
`BigBen` employs an extensive override system to allow someone to override
the default properties. The order of priority is system properties > system env variables >
overrides > defaults
The overrides can be defined in `config/overrides.yaml` file.
The `log4j.xml` can also be changed to change log behavior without
recompiling binaries
## How to setup `Cassandra` for `BigBen`?
Following are the steps to set up `Cassandra`:
1. git clone the `master` branch
2. Set up a Cassandra cluster
3. create a keyspace `bigben` in `Cassandra` cluster with desired replication
4. Open the file `bigben-schema.cql` and execute `cqlsh -f bigben-schema.cql`
## APIs
### cluster
`GET /events/cluster`
* response sample (a 3 node cluster running on single machine and three different ports (5701, 5702, 5703)):
```json
{
"[127.0.0.1]:5702": "Master",
"[127.0.0.1]:5701": "Slave",
"[127.0.0.1]:5703": "Slave"
}
```
The node marked `Master` is the master node that does the scheduling.
### tenant registration
A tenant can be registered by calling the following API
`POST /events/tenant/register`
* payload schema
```json
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"tenant": {
"type": "string"
},
"type": {
"type": "string"
},
"props": {
"type": "object"
}
},
"required": [
"tenant",
"type",
"props"
]
}
```
* `tenant` - specifies a tenant and can be any arbitrary value.
* `type` - specifies the type of `tenant`. One of the three types can be used
* MESSAGING - specifies that `tenant` wants events delivered via a messaging queue. Currently, `kafka`
is the only supported messaging system.
* HTTP - specifies that `tenant` wants events delivered via an http callback URL.
* CUSTOM_CLASS - specifies a custom event processor implemented for custom processing of events
* `props` - A bag of properties needed for each type of tenant.
* kafka sample:
```json
{
"tenant": "TenantA/ProgramB/EnvC",
"type": "MESSAGING",
"props": {
"topic": "some topic name",
"bootstrap.servers": "node1:9092,node2:9092"
}
}
```
* http sample
```json
{
"tenant": "TenantB/ProgramB/EnvC",
"type": "HTTP",
"props": {
"url": "http://someurl",
"headers": {
"header1": "value1",
"header2": "value2"
}
}
}
```
### fetch all tenants:
`GET /events/tenants`
### event scheduling
`POST /events/schedule`
`Payload - List<EventRequest>`
`EventRequest` schema:
```json
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"id": {
"type": "string"
},
"eventTime": {
"type": "string",
"description": "An ISO-8601 formatted timestamp e.g. 2018-01-31T04:00.00Z"
},
"tenant": {
"type": "string"
},
"payload": {
"type": "string",
"description": "an optional event payload, must NOT be null with deliveryOption = PAYLOAD_ONLY"
},
"mode": {
"type": "string",
"enum": ["UPSERT", "REMOVE"],
"default": "UPSERT",
"description": "Use REMOVE to delete an event, UPSERT to add/update an event"
},
"deliveryOption": {
"type": "string",
"enum": ["FULL_EVENT", "PAYLOAD_ONLY"],
"default": "FULL_EVENT",
"description": "Use FULL_EVENT to have full event delivered via kafka/http, PAYLOAD_ONLY to have only the payload delivered"
}
},
"required": [
"id",
"eventTime",
"tenant"
]
}
```
### find an event
`GET /events/find?id=?&tenant=?`
### dry run
`POST /events/dryrun?id=?&tenant=?`
fires an event without changing its final status
## cron APIs
coming up...
================================================
FILE: app/LICENSE.txt
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2018 Sandeep Malik
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: app/pom.xml
================================================
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.walmartlabs.bigben</groupId>
<artifactId>bigben</artifactId>
<version>1.0.7-SNAPSHOT</version>
</parent>
<artifactId>bigben-app</artifactId>
<name>BigBen:app</name>
<url>http://maven.apache.org</url>
<packaging>takari-jar</packaging>
<properties>
<slf4j-api.version>1.7.25</slf4j-api.version>
</properties>
<dependencies>
<dependency>
<groupId>com.walmartlabs.bigben</groupId>
<artifactId>bigben-lib</artifactId>
</dependency>
<dependency>
<groupId>com.walmartlabs.bigben</groupId>
<artifactId>bigben-cassandra</artifactId>
</dependency>
<dependency>
<groupId>com.walmartlabs.bigben</groupId>
<artifactId>bigben-kafka</artifactId>
</dependency>
<dependency>
<groupId>com.walmartlabs.bigben</groupId>
<artifactId>bigben-cron</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j-api.version}</version>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-server-core</artifactId>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-server-netty</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http2</artifactId>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-jackson</artifactId>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-client-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.ktor</groupId>
<artifactId>ktor-client-apache</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<buildDirectory>${project.build.directory}</buildDirectory>
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<finalName>bigben</finalName>
<appendAssemblyId>false</appendAssemblyId>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<mainClass>com.walmartlabs.bigben.app.RunKt</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>assemble-all</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
================================================
FILE: app/src/main/kotlin/com/walmartlabs/bigben/app/app.kt
================================================
/*-
* #%L
* BigBen:app
* =======================================
* Copyright (C) 2016 - 2018 Walmart Inc.
* =======================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.walmartlabs.bigben.app
/**
* Created by smalik3 on 2/28/18
*/
import com.walmartlabs.bigben.BigBen
import com.walmartlabs.bigben.BigBen.module
import com.walmartlabs.bigben.api.EventReceiver
import com.walmartlabs.bigben.entities.EventRequest
import com.walmartlabs.bigben.entities.EventStatus.REJECTED
import com.walmartlabs.bigben.extns.bucket
import com.walmartlabs.bigben.extns.nowUTC
import com.walmartlabs.bigben.utils.*
import com.walmartlabs.bigben.utils.commons.Module
import com.walmartlabs.bigben.utils.commons.ModuleRegistry
import org.slf4j.Logger
import java.time.Duration
import java.time.ZonedDateTime
import java.time.temporal.ChronoUnit
import java.util.*
import java.util.concurrent.ThreadLocalRandom
import java.util.concurrent.TimeUnit
import kotlin.system.exitProcess
class App {
init {
try {
val lifecycle =
typeRefYaml<Map<String, String?>>(App::class.java.classLoader.getResource("bigben-lifecycle.yaml").readText())
initPhase("pre-init", lifecycle, null)
val l = logger<App>()
l.info("phase:pre-init finished")
println(
"\n" +
" ____ _ ____ \n" +
" | _ \\(_) | _ \\ \n" +
" | |_) |_ __ _| |_) | ___ _ __ \n" +
" | _ <| |/ _` | _ < / _ \\ '_ \\ \n" +
" | |_) | | (_| | |_) | __/ | | |\n" +
" |____/|_|\\__, |____/ \\___|_| |_|\n" +
" __/ | \n" +
" |___/ \n"
)
BigBen.init()
initPhase("post-init", lifecycle, l)
l.info("Bigben => successfully started")
} catch (e: Exception) {
try {
val l: Logger = logger<App>()
l.error("Bigben:error => unknown error, system will exit", e.rootCause()!!)
} catch (ignore: Exception) {
}
exitProcess(1)
}
}
private fun initPhase(phase: String, lifecycle: Map<String, String?>, l: Logger?) {
l?.info("phase:$phase started")
lifecycle["$phase-class"]?.run { (Class.forName(this).newInstance() as Module).init(ModuleRegistry()) }
?: lifecycle["$phase-object"]?.run {
(Class.forName(this).getDeclaredField("INSTANCE").apply {
isAccessible = true
}.get(null) as Module).init(ModuleRegistry())
}
l?.info("phase:$phase finished")
}
}
object EventGenerator {
data class EventGeneration(val offset: String, val period: String, val numEvents: Int, val tenant: String)
private val l = logger<EventGenerator>()
fun generateEvents(eg: EventGeneration): Map<ZonedDateTime, Int> {
val random = ThreadLocalRandom.current()
val t1 = nowUTC().bucket() + Duration.parse(eg.offset)
val t2 = t1 + Duration.parse(eg.period)
val delta = ChronoUnit.MILLIS.between(t1, t2)
l.info("generating ${eg.numEvents} random events between $t1 and $t2")
return (1..eg.numEvents).map {
val t = if(delta > 0) t1.plus(random.nextLong(delta), ChronoUnit.MILLIS) else t1
module<EventReceiver>().addEvent(EventRequest().also {
it.tenant = eg.tenant
it.eventTime = t.toString()
it.id = UUID.randomUUID().toString()
}).transform { if (it?.eventStatus == REJECTED) throw IllegalArgumentException(it.error?.message) else it }
}.reduce().transform {
it!!.groupBy { ZonedDateTime.parse(it!!.eventTime).bucket() }.mapValues { it.value.size }.toSortedMap()
}.get(30L, TimeUnit.MINUTES)
}
}
================================================
FILE: app/src/main/kotlin/com/walmartlabs/bigben/app/run.kt
================================================
package com.walmartlabs.bigben.app
import com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT
import com.walmartlabs.bigben.BigBen.module
import com.walmartlabs.bigben.api.EventService
import com.walmartlabs.bigben.cron.CronService
import com.walmartlabs.bigben.extns.APIResponse
import com.walmartlabs.bigben.utils.stackTraceAsString
import com.walmartlabs.bigben.utils.typeRefJson
import io.ktor.application.Application
import io.ktor.application.ApplicationCall
import io.ktor.application.call
import io.ktor.application.install
import io.ktor.features.ContentNegotiation
import io.ktor.features.StatusPages
import io.ktor.http.HttpStatusCode
import io.ktor.http.HttpStatusCode.Companion.BadRequest
import io.ktor.http.HttpStatusCode.Companion.InternalServerError
import io.ktor.jackson.jackson
import io.ktor.request.receive
import io.ktor.response.header
import io.ktor.response.respond
import io.ktor.routing.*
import io.ktor.server.netty.EngineMain
import org.apache.commons.text.StrLookup
import org.apache.commons.text.StrSubstitutor
import org.apache.log4j.xml.DOMConfigurator
import java.io.File
fun main(args: Array<String>) = EngineMain.main(args)
fun logs() {
System.getProperty("bigben.log.config")?.run {
val logFile = File(this)
if (logFile.exists()) {
println("configuring logger")
StrSubstitutor(StrLookup.systemPropertiesLookup()).run {
logFile.readLines().map { replace(it) }
}.joinToString("\n").run {
File(System.getProperty("java.io.tmpdir"), "log4j-overrides-substituted.xml").let {
println("using log file from ${it.absolutePath}")
it.writeText(this)
DOMConfigurator.configure(it.toURI().toURL())
}
}
}
}
}
fun app() = App()
fun Application.routes() {
routing {
get("/ping") { call.respond(mapOf("status" to "OK")) }
route("/events") {
val es = module<EventService>()
get("/cluster") { call.fromAPIResponse(es.clusterStats()) }
post("/schedule") { call.fromAPIResponse(es.schedule(typeRefJson(call.receive()))) }
post("/tenant/register") { call.fromAPIResponse(es.registerProcessor(call.receive())) }
get("/tenants") { call.fromAPIResponse(es.registeredTenants()) }
get("/find") { call.fromAPIResponse(es.find(call.request.queryParameters["id"]!!, call.request.queryParameters["tenant"]!!)) }
post("/dryrun") { call.fromAPIResponse(es.dryrun(call.request.queryParameters["id"]!!, call.request.queryParameters["tenant"]!!)) }
}
post("/generation/random") { call.respond(EventGenerator.generateEvents(call.receive())) }
route("/cron") {
post { call.fromAPIResponse(CronService.upsert(call.receive())) }
get("/describe") { call.fromAPIResponse(CronService.describe(call.receive())) }
get("/{tenant}/{id}") {
call.fromAPIResponse(
CronService.get(
call.parameters["tenant"]!!, call.parameters["id"]!!,
call.request.queryParameters["describe"]?.toBoolean()
)
)
}
delete("/{tenant}/{id}/{type}") {
call.fromAPIResponse(CronService.delete(call.parameters["tenant"]!!, call.parameters["id"]!!, call.parameters["type"]!!))
}
}
}
}
fun Application.configure() {
install(ContentNegotiation) {
jackson { enable(INDENT_OUTPUT) }
}
install(StatusPages) {
exception<IllegalArgumentException> { e ->
call.response.status(BadRequest)
call.respond(mapOf("message" to (e.message ?: "")))
}
exception<Throwable> { e ->
call.response.status(InternalServerError)
if (call.request.queryParameters["debug"] != null) {
call.respond(mapOf("message" to ((e.message ?: "")), "stacktrace" to e.stackTraceAsString()))
} else call.respond(mapOf("message" to (e.message ?: "")))
}
}
}
private suspend fun ApplicationCall.fromAPIResponse(r: APIResponse) {
r.headers.forEach { h -> h.value.forEach { response.header(h.key, it) } }
response.status(HttpStatusCode.fromValue(r.status))
respond(r.entity)
}
================================================
FILE: app/src/main/resources/application.conf
================================================
ktor {
deployment {
port = 8080
port = ${?app.server.port}
}
application {
modules = [
com.walmartlabs.bigben.app.RunKt.logs
com.walmartlabs.bigben.app.RunKt.configure
com.walmartlabs.bigben.app.RunKt.app
com.walmartlabs.bigben.app.RunKt.routes
]
}
}
================================================
FILE: app/src/main/resources/bigben-lifecycle.yaml
================================================
pre-init-class: null
post-init-class: null
================================================
FILE: app/src/main/resources/bigben.yaml
================================================
# top level modules
modules:
- name: domain
class: com.walmartlabs.bigben.providers.domain.cassandra.CassandraModule
- name: processors
object: com.walmartlabs.bigben.processors.ProcessorRegistry
- name: hz
class: com.walmartlabs.bigben.utils.hz.Hz
- name: scheduler
object: com.walmartlabs.bigben.SchedulerModule
- name: events
object: com.walmartlabs.bigben.EventModule
- name: messaging
object: com.walmartlabs.bigben.kafka.KafkaModule
enabled: ${kafka.module.enabled:-false}
- name: cron
object: com.walmartlabs.bigben.cron.CronRunner
enabled: ${cron.module.enabled:-false}
# hazelcast properties
hz:
template: file://hz.template.xml
group:
name: bigben-dev
password: bigben-dev
network:
autoIncrementPort: true
members: 127.0.0.1
port: 5701
map:
store:
writeDelay: 30
# message related properties
messaging.producer.factory.class: com.walmartlabs.bigben.kafka.KafkaMessageProducerFactory
# cassandra related properties
cassandra:
keyspace: bigben
cluster:
contactPoints: 127.0.0.1
clusterName: bigben-cluster
port: 9042
localDataCenter: null
coreConnectionsPerLocalHost: 1
maxConnectionsPerLocalHost: 1
coreConnectionsPerRemoteHost: 1
maxConnectionsPerRemoteHost: 1
maxRequestsPerLocalConnection: 32768
maxRequestsPerRemoteConnection: 2048
newLocalConnectionThreshold: 3000
newRemoteConnectionThreshold: 400
poolTimeoutMillis: 0
keepTCPConnectionAlive: true
connectionTimeOut: 5000
readTimeout: 12000
reconnectPeriod: 5
username: null
password: null
downgradingConsistency: false
writeConsistency: LOCAL_ONE
readConsistency: LOCAL_ONE
# kafka consumer properties
kafka:
consumers:
- num.consumers: ${num.consumers:-8}
processor.impl.class: com.walmartlabs.bigben.kafka.ProcessorImpl
topics: ${bigben.inbound.topic.name:-null}
max.poll.wait.time: ${max.poll.wait.time:-10000}
message.retry.max.count: ${message.retry.max.count:-10}
config:
key.deserializer: org.apache.kafka.common.serialization.StringDeserializer
value.deserializer: org.apache.kafka.common.serialization.StringDeserializer
bootstrap.servers: ${bigben.inbound.topic.bootstrap.servers:-null}
#fetch.min.bytes: 1
group.id: ${group.id:-bigben-inbound}
heartbeat.interval.ms: ${heartbeat.interval.ms:-3000}
session.timeout.ms: 30000
auto.offset.reset: ${auto.offset.reset:-latest}
fetch.max.bytes: 324000
max.poll.interval.ms: 30000
max.poll.records: 100
receive.buffer.bytes: 65536
request.timeout.ms: 60000
#send.buffer.bytes: 131072
enable.auto.commit: ${enable.auto.commit:-false}
producer:
config: # this is default kafka producer config, these values will be used if not supplied during the tenant registration
key.serializer: org.apache.kafka.common.serialization.StringSerializer
value.serializer: org.apache.kafka.common.serialization.StringSerializer
acks: "1"
buffer.memory: 32400
retries: 3
# system properties
task:
executor:
#retry.thread.count: 8
retry.time.units: SECONDS
delay: 1
max.retries: 3
backoff.multiplier: 2
app.server.port: 8080
generic.future.max.get.time: 60
events:
scheduler.enabled: true
schedule.scan.interval.minutes: 1
num.shard.submitters: 8
receiver:
shard.size: 1000
lapse.offset.minutes: 0
delete:
max.retries: 3
initial.delay: 1
backoff.multiplier: 1
submit:
initial.delay: 1
backoff.multiplier: 1
max.retries: 3
processor:
max.retries: 3
initial.delay: 1
backoff.multiplier: 2
eager.loading: true
tasks:
max.events.in.memory: 100000
scheduler.worker.threads: 8
# bucket manager / loader related properties
buckets:
backlog.check.limit: 1440 # 1 Day
background:
load.fetch.size: 100
load.wait.interval.seconds: 15
cron:
runner:
core.pool.size: 8
load:
max.retries: 10
delay: 1
backoff.multiplier: 1
time.units: "SECONDS"
================================================
FILE: app/src/main/resources/log4j.xml
================================================
<?xml version="1.0" encoding="UTF-8" ?>
<!--
#%L
BigBen:app
=======================================
Copyright (C) 2016 - 2018 Walmart Inc.
=======================================
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
#L%
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration debug="true"
xmlns:log4j='http://jakarta.apache.org/log4j/'
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://jakarta.apache.org/log4j/ ">
<appender name="console" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%t] %c{1}:%L - %m%n"/>
</layout>
</appender>
<root>
<level value="WARN"/>
<appender-ref ref="console"/>
</root>
<logger name="com.walmartlabs.bigben" additivity="false">
<level value="INFO"/>
<appender-ref ref="console"/>
</logger>
</log4j:configuration>
================================================
FILE: app/src/test/kotlin/com/walmartlabs/bigben/tests/APITests.kt
================================================
package com.walmartlabs.bigben.tests
import com.datastax.driver.core.Session
import com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT
import com.walmartlabs.bigben.BigBen
import com.walmartlabs.bigben.BigBen.module
import com.walmartlabs.bigben.app.EventGenerator
import com.walmartlabs.bigben.app.main
import com.walmartlabs.bigben.entities.EntityProvider
import com.walmartlabs.bigben.entities.EventDeliveryOption.FULL_EVENT
import com.walmartlabs.bigben.entities.EventDeliveryOption.PAYLOAD_ONLY
import com.walmartlabs.bigben.entities.EventLoader
import com.walmartlabs.bigben.entities.EventRequest
import com.walmartlabs.bigben.entities.EventResponse
import com.walmartlabs.bigben.entities.EventStatus.*
import com.walmartlabs.bigben.extns.nowUTC
import com.walmartlabs.bigben.kafka.MockMessageProducerFactory
import com.walmartlabs.bigben.processors.ProcessorConfig
import com.walmartlabs.bigben.processors.ProcessorConfig.Type.*
import com.walmartlabs.bigben.utils.fromJson
import com.walmartlabs.bigben.utils.json
import com.walmartlabs.bigben.utils.stackTraceAsString
import com.walmartlabs.bigben.utils.typeRefJson
import io.ktor.application.call
import io.ktor.application.install
import io.ktor.client.HttpClient
import io.ktor.client.call.call
import io.ktor.client.engine.apache.Apache
import io.ktor.client.request.accept
import io.ktor.client.request.post
import io.ktor.client.request.url
import io.ktor.client.response.readText
import io.ktor.content.TextContent
import io.ktor.features.ContentNegotiation
import io.ktor.features.StatusPages
import io.ktor.http.ContentType.Application.Json
import io.ktor.http.HttpMethod
import io.ktor.http.HttpMethod.Companion.Get
import io.ktor.http.HttpMethod.Companion.Post
import io.ktor.http.HttpStatusCode
import io.ktor.http.HttpStatusCode.Companion.OK
import io.ktor.jackson.jackson
import io.ktor.request.contentType
import io.ktor.request.header
import io.ktor.request.receive
import io.ktor.response.respond
import io.ktor.routing.post
import io.ktor.routing.routing
import io.ktor.server.engine.embeddedServer
import io.ktor.server.netty.Netty
import kotlinx.coroutines.runBlocking
import org.testng.annotations.AfterClass
import org.testng.annotations.BeforeClass
import org.testng.annotations.Test
import java.time.ZonedDateTime
import java.util.UUID.randomUUID
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.SynchronousQueue
import kotlin.concurrent.thread
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class APITests {
companion object {
init {
System.setProperty("bigben.configs", "file://bigben-api-test.yaml")
thread { main(emptyArray()) }
BigBen.init()
}
}
private val client = HttpClient(Apache)
private val server = "http://localhost:8080"
private val responses = ConcurrentHashMap<String, SynchronousQueue<EventResponse>>()
private val payloadResponse = SynchronousQueue<String>()
@BeforeClass
private fun `clean up db`() {
println("cleaning up the db")
(module<EntityProvider<Any>>().unwrap() as Session).apply {
execute("truncate bigben.events;")
execute("truncate bigben.lookups;")
execute("truncate bigben.buckets;")
execute("truncate bigben.kv_table;")
}
thread {
embeddedServer(Netty, 9090) {
println("starting test server")
install(ContentNegotiation) {
jackson { enable(INDENT_OUTPUT) }
}
install(StatusPages) {
exception<Throwable> { e ->
//e.printStackTrace()
call.response.status(HttpStatusCode.InternalServerError)
call.respond(mapOf("message" to ((e.message ?: "")), "stacktrace" to e.stackTraceAsString()))
}
}
routing {
post("/test") {
if (call.request.queryParameters["error"] != null) {
throw IllegalArgumentException("test error")
}
val er = call.receive<EventResponse>()
assertEquals(call.request.header("header1"), "value1")
assertEquals(call.request.header("header2"), "value2")
assertEquals(call.request.contentType().contentType, Json.contentType)
assertEquals(call.request.contentType().contentSubtype, Json.contentSubtype)
responses[er.id!!]!!.put(er)
call.respond(OK, mapOf("status" to "OK"))
}
post("/payload") {
if (call.request.queryParameters["error"] != null) {
throw IllegalArgumentException("test error")
}
val payload = call.receive<String>()
assertEquals(call.request.header("header1"), "value1")
assertEquals(call.request.header("header2"), "value2")
assertEquals(call.request.contentType().contentType, Json.contentType)
assertEquals(call.request.contentType().contentSubtype, Json.contentSubtype)
payloadResponse.put(payload)
call.respond(OK, mapOf("status" to "OK"))
}
}
}.start(true)
}
}
@AfterClass
fun teardown() {
client.close()
}
@Test(enabled = true)
fun `test events at the same time`() {
val tenant = "test"
assertEquals(runBlocking {
client.call {
url("$server/events/tenant/register")
accept(Json)
method = Post
body = TextContent(
ProcessorConfig(
tenant, CUSTOM_CLASS,
mapOf("eventProcessorClass" to "com.walmartlabs.bigben.processors.NoOpCustomClassProcessor")
).json(), Json
)
}.response.status.value
}, 200)
// schedule 1000 events at exactly same time at the start of the minute:
runBlocking {
client.post<String> {
url("$server/generation/random")
accept(Json)
body = TextContent(EventGenerator.EventGeneration("PT1M", "PT0S", 1000, tenant).json(), Json)
}
}
// schedule 1000 events at exactly same time at the start of the minute + 30 seconds:
val bucket = runBlocking {
client.post<String> {
url("$server/generation/random")
accept(Json)
body = TextContent(EventGenerator.EventGeneration("PT1M30S", "PT0S", 1000, tenant).json(), Json)
}
}.run { typeRefJson<Map<String, Int>>(this).run { ZonedDateTime.parse(entries.first().key) } }
Thread.sleep(2 * 60 * 1000) // sleep for 2 minutes
var total = 0
(0..1).forEach {
// 2000 events -> 2 shards
var l = module<EventLoader>().load(bucket, 0, 400).get()
while (l.second.isNotEmpty()) {
l.second.forEach {
assertEquals(it.bucketId, bucket)
assertEquals(it.status, PROCESSED)
assertTrue { it.eventTime == bucket || it.eventTime == bucket.plusSeconds(30) }
total++
}
l = module<EventLoader>().load(bucket, 0, 400, l.second.last().eventTime!!, l.second.last().id!!, l.first)
.get()
}
}
}
@Test
fun `test unknown tenant rejection`() {
val (status, content) = client.call("/events/schedule", listOf(EventRequest("id123", nowUTC().toString(), "ABC")))
assertEquals(status, 400)
assertEquals(typeRefJson<List<EventResponse>>(content)[0].eventStatus, REJECTED)
}
@Test
fun `test missing tenant`() {
val (status, content) = client.call("/events/schedule", listOf(EventRequest("id123", nowUTC().toString())))
assertEquals(status, 400)
assertEquals(typeRefJson<List<EventResponse>>(content)[0].eventStatus, REJECTED)
}
@Test
fun `test missing event time`() {
val (status, content) = client.call("/events/schedule", listOf(EventRequest("id123", tenant = "ABC")))
assertEquals(status, 400)
assertEquals(typeRefJson<List<EventResponse>>(content)[0].eventStatus, REJECTED)
}
@Test
fun `test event time wrong format`() {
val (status, content) = client.call("/events/schedule", listOf(EventRequest("id123", "time", tenant = "ABC")))
assertEquals(status, 400)
assertEquals(typeRefJson<List<EventResponse>>(content)[0].eventStatus, REJECTED)
}
@Test
fun `test event time in past`() {
val tenant = "http"
assertEquals(
client.call(
"/events/tenant/register", ProcessorConfig(
tenant, HTTP, mapOf(
"url" to "http://localhost:9090/test", "headers" to mapOf("header1" to "value1", "header2" to "value2")
)
)
).first, 200
)
val eventId = "id123"
responses[eventId] = SynchronousQueue()
val (status, content) = client.call("/events/schedule", listOf(EventRequest(eventId, nowUTC().minusMinutes(1).toString(), tenant)))
assertEquals(status, 200)
assertEquals(typeRefJson<List<EventResponse>>(content)[0].eventStatus, TRIGGERED)
val er = responses[eventId]!!.take()
assertEquals(er.eventStatus, TRIGGERED)
assertEquals(er.deliveryOption, FULL_EVENT)
assertEquals(er.tenant!!, tenant)
}
@Test
fun `test event time in past - payload only`() {
val tenant = "http3"
assertEquals(
client.call(
"/events/tenant/register", ProcessorConfig(
tenant, HTTP, mapOf(
"url" to "http://localhost:9090/payload", "headers" to mapOf("header1" to "value1", "header2" to "value2")
)
)
).first, 200
)
val eventId = "id234"
responses[eventId] = SynchronousQueue()
val (status, content) = client.call("/events/schedule", listOf(EventRequest(eventId, nowUTC().minusMinutes(1).toString(), tenant, payload = "testP", deliveryOption = PAYLOAD_ONLY)))
assertEquals(status, 200)
assertEquals(typeRefJson<List<EventResponse>>(content)[0].eventStatus, TRIGGERED)
val payload = payloadResponse.take()
assertEquals(payload, "testP")
}
@Test
fun `test event - null payload with payload only option`() {
val tenant = "http2"
assertEquals(
client.call(
"/events/tenant/register", ProcessorConfig(
tenant, HTTP, mapOf(
"url" to "http://localhost:9090/test", "headers" to mapOf("header1" to "value1", "header2" to "value2")
)
)
).first, 200
)
val eventId = randomUUID().toString()
responses[eventId] = SynchronousQueue()
val (status, content) = client.call("/events/schedule", listOf(EventRequest(eventId, nowUTC().plusMinutes(1).toString(), tenant, deliveryOption = PAYLOAD_ONLY)))
assertEquals(status, 400)
assertEquals(typeRefJson<List<EventResponse>>(content)[0].eventStatus, REJECTED)
}
@Test
fun `test find and dryrun APIs`() {
val tenant = "http4"
assertEquals(
client.call(
"/events/tenant/register", ProcessorConfig(
tenant, HTTP, mapOf(
"url" to "http://localhost:9090/test", "headers" to mapOf("header1" to "value1", "header2" to "value2")
)
)
).first, 200
)
val eventId = randomUUID().toString()
responses[eventId] = SynchronousQueue()
val eventTime = nowUTC().plusMinutes(100000).toString()
val (status, content) = client.call("/events/schedule", listOf(EventRequest(eventId, eventTime, tenant, payload = "P1")))
assertEquals(status, 200)
assertEquals(typeRefJson<List<EventResponse>>(content)[0].eventStatus, ACCEPTED)
val (s, c) = client.call("/events/find?tenant=$tenant&id=$eventId", null, Get)
assertEquals(s, 200)
val er = EventResponse::class.java.fromJson(c)
assertEquals(er.eventStatus, UN_PROCESSED)
assertEquals(er.payload, "P1")
assertEquals(er.tenant, tenant)
assertFalse { er.eventId!!.startsWith("a-") }
assertEquals(er.deliveryOption, FULL_EVENT)
assertEquals(er.eventTime, eventTime)
val (s1, c1) = client.call("/events/dryrun?tenant=$tenant&id=$eventId", null)
assertEquals(s1, 200)
val er1 = EventResponse::class.java.fromJson(c1)
assertEquals(er1.eventStatus, UN_PROCESSED)
assertEquals(er1.payload, "P1")
assertEquals(er1.tenant, tenant)
assertFalse { er1.eventId!!.startsWith("a-") }
assertEquals(er1.deliveryOption, FULL_EVENT)
assertEquals(er1.eventTime, eventTime)
responses[eventId]!!.take().apply {
assertEquals(eventStatus, TRIGGERED)
assertEquals(payload, "P1")
assertEquals(this.tenant, tenant)
assertFalse { this.eventId!!.startsWith("a-") }
assertEquals(deliveryOption, FULL_EVENT)
assertEquals(eventTime, eventTime)
}
}
@Test
fun `test find and dryrun APIs - payload only`() {
val tenant = "http5"
assertEquals(
client.call(
"/events/tenant/register", ProcessorConfig(
tenant, HTTP, mapOf(
"url" to "http://localhost:9090/payload", "headers" to mapOf("header1" to "value1", "header2" to "value2")
)
)
).first, 200
)
val eventId = randomUUID().toString()
responses[eventId] = SynchronousQueue()
val eventTime = nowUTC().plusMinutes(100000).toString()
val (status, content) = client.call("/events/schedule", listOf(EventRequest(eventId, eventTime, tenant, payload = "P1", deliveryOption = PAYLOAD_ONLY)))
assertEquals(status, 200)
assertEquals(typeRefJson<List<EventResponse>>(content)[0].eventStatus, ACCEPTED)
val (s, c) = client.call("/events/find?tenant=$tenant&id=$eventId", null, Get)
assertEquals(s, 200)
val er = EventResponse::class.java.fromJson(c)
assertEquals(er.eventStatus, UN_PROCESSED)
assertEquals(er.payload, "P1")
assertEquals(er.tenant, tenant)
assertTrue { er.eventId!!.startsWith("a-") }
assertEquals(er.deliveryOption, PAYLOAD_ONLY)
assertEquals(er.eventTime, eventTime)
val (s1, c1) = client.call("/events/dryrun?tenant=$tenant&id=$eventId", null)
assertEquals(s1, 200)
val er1 = EventResponse::class.java.fromJson(c1)
assertEquals(er1.eventStatus, UN_PROCESSED)
assertEquals(er1.payload, "P1")
assertEquals(er1.tenant, tenant)
assertTrue { er1.eventId!!.startsWith("a-") }
assertEquals(er1.deliveryOption, PAYLOAD_ONLY)
assertEquals(er1.eventTime, eventTime)
payloadResponse.take().apply {
assertEquals(this, "P1")
}
}
@Test
fun `test event kafka tenant API`() {
val tenant = "kafka1"
assertEquals(
client.call(
"/events/tenant/register", ProcessorConfig(
tenant, MESSAGING, mapOf(
"topic" to "topic1",
"bootstrap.servers" to "localhost:9092"
)
)
).first, 200
)
val eventId = randomUUID().toString()
responses[eventId] = SynchronousQueue()
val eventTime = nowUTC().minusMinutes(1).toString()
val (status, content) = client.call("/events/schedule", listOf(EventRequest(eventId, eventTime, tenant, payload = "P1")))
assertEquals(status, 200)
assertEquals(typeRefJson<List<EventResponse>>(content)[0].eventStatus, TRIGGERED)
Thread.sleep(2000)
val er = MockMessageProducerFactory.LAST_MESSAGE.get()
assertEquals(er.eventStatus, TRIGGERED)
assertEquals(er.payload, "P1")
assertEquals(er.tenant, tenant)
assertEquals(er.deliveryOption, FULL_EVENT)
assertEquals(er.eventTime, eventTime)
}
@Test
fun `test event kafka tenant API - payload only`() {
val tenant = "kafka2"
assertEquals(
client.call(
"/events/tenant/register", ProcessorConfig(
tenant, MESSAGING, mapOf(
"topic" to "topic2",
"bootstrap.servers" to "localhost:9092"
)
)
).first, 200
)
val eventId = randomUUID().toString()
responses[eventId] = SynchronousQueue()
val eventTime = nowUTC().minusMinutes(1).toString()
val (status, content) = client.call("/events/schedule", listOf(EventRequest(eventId, eventTime, tenant, payload = "P2", deliveryOption = PAYLOAD_ONLY)))
assertEquals(status, 200)
assertEquals(typeRefJson<List<EventResponse>>(content)[0].eventStatus, TRIGGERED)
val er = MockMessageProducerFactory.LAST_MESSAGE.get()
assertEquals(er.eventStatus, TRIGGERED)
assertEquals(er.payload, "P2")
assertEquals(er.tenant, tenant)
assertEquals(er.deliveryOption, PAYLOAD_ONLY)
assertEquals(er.eventTime, eventTime)
}
private fun HttpClient.call(url: String, body: Any?, method: HttpMethod = Post): Pair<Int, String> {
return runBlocking {
client.call {
url("$server$url")
accept(Json)
this.method = method
body?.let { this.body = TextContent(it.json(), Json) }
}.response.run { status.value to this.readText().apply { println("response: $this") } }
}
}
}
================================================
FILE: app/src/test/kotlin/com/walmartlabs/bigben/tests/BigBenTests.kt
================================================
/*-
* #%L
* BigBen:app
* =======================================
* Copyright (C) 2016 - 2018 Walmart Inc.
* =======================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.walmartlabs.bigben.tests
import com.datastax.driver.core.Session
import com.google.common.util.concurrent.ListeningScheduledExecutorService
import com.sun.net.httpserver.HttpServer
import com.walmartlabs.bigben.BigBen
import com.walmartlabs.bigben.BigBen.module
import com.walmartlabs.bigben.api.EventService
import com.walmartlabs.bigben.core.BucketManager
import com.walmartlabs.bigben.core.BucketsLoader
import com.walmartlabs.bigben.core.ScheduleScanner
import com.walmartlabs.bigben.entities.*
import com.walmartlabs.bigben.entities.EventStatus.*
import com.walmartlabs.bigben.entities.Mode.REMOVE
import com.walmartlabs.bigben.extns.bucket
import com.walmartlabs.bigben.extns.fetch
import com.walmartlabs.bigben.extns.nowUTC
import com.walmartlabs.bigben.extns.save
import com.walmartlabs.bigben.processors.NoOpCustomClassProcessor
import com.walmartlabs.bigben.processors.ProcessorConfig
import com.walmartlabs.bigben.processors.ProcessorConfig.Type.*
import com.walmartlabs.bigben.utils.commons.Props.int
import com.walmartlabs.bigben.utils.commons.Props.map
import com.walmartlabs.bigben.utils.commons.TaskExecutor
import com.walmartlabs.bigben.utils.fromJson
import com.walmartlabs.bigben.utils.json
import org.apache.commons.text.RandomStringGenerator
import org.apache.kafka.clients.consumer.KafkaConsumer
import org.apache.kafka.clients.producer.KafkaProducer
import org.apache.kafka.clients.producer.ProducerRecord
import org.testng.annotations.BeforeClass
import org.testng.annotations.BeforeMethod
import org.testng.annotations.Test
import java.lang.Thread.sleep
import java.net.InetSocketAddress
import java.time.ZonedDateTime
import java.util.*
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ScheduledThreadPoolExecutor
import java.util.concurrent.TimeUnit.MINUTES
import java.util.concurrent.atomic.AtomicInteger
import kotlin.reflect.full.memberProperties
import kotlin.reflect.jvm.isAccessible
import kotlin.test.assertEquals
import kotlin.test.assertTrue
/**
* Created by smalik3 on 4/11/18
*/
class BigBenTests {
companion object {
init {
System.setProperty("bigben.configs", "file://bigben-test.yaml,file://bigben.yaml")
BigBen.init()
}
private val eventService = BigBen.module<EventService>()
}
@BeforeClass
fun `set up tenant`() {
`clean up db`()
eventService.registerProcessor(
ProcessorConfig(
"default", CUSTOM_CLASS,
mapOf("eventProcessorClass" to NoOpCustomClassProcessor::class.java.name)
)
).apply { assertEquals(this.status, 200) }
println("tenant set up done")
}
@BeforeMethod
private fun `clean up db`() {
println("cleaning up the db")
(module<EntityProvider<Any>>().unwrap() as Session).apply {
execute("truncate bigben.events;")
execute("truncate bigben.lookups;")
execute("truncate bigben.buckets;")
execute("truncate bigben.kv_table;")
}
}
@Test
fun `event service schedule and find API`() {
val eventTime = nowUTC().plusMinutes(3)
val tenant = "default"
val xrefId = "abc"
//add:
eventService.schedule(listOf(EventRequest(xrefId, eventTime.toString(), tenant, "P"))).apply {
assertEquals(status, 200)
}
eventService.find(xrefId, tenant).apply {
assertEquals(status, 200)
(entity as EventResponse).apply {
assertEquals(ZonedDateTime.parse(this.eventTime), eventTime)
assertEquals(payload, "P")
}
}
//update payload:
eventService.schedule(listOf(EventRequest(xrefId, eventTime.toString(), tenant, "P1"))).apply {
assertEquals(status, 200)
}
eventService.find(xrefId, tenant).apply {
assertEquals(status, 200)
(entity as EventResponse).apply {
assertEquals(ZonedDateTime.parse(this.eventTime), eventTime)
assertEquals(payload, "P1")
}
}
// update time:
eventService.schedule(listOf(EventRequest(xrefId, eventTime.plusMinutes(1).toString(), tenant, "P2"))).apply {
assertEquals(status, 200)
}
eventService.find(xrefId, tenant).apply {
assertEquals(status, 200)
(entity as EventResponse).apply {
assertEquals(ZonedDateTime.parse(this.eventTime), eventTime.plusMinutes(1))
assertEquals(payload, "P2")
}
}
//remove event:
eventService.schedule(listOf(EventRequest(xrefId, eventTime.plusMinutes(1).toString(), tenant, "P2", REMOVE)))
.apply {
assertEquals(status, 200)
}
eventService.find(xrefId, tenant).apply {
assertEquals(status, 404)
}
}
@Test
fun `sharding works as expected`() {
val r = Random()
val time = nowUTC().plusMinutes(2).bucket()
(0..100).forEach {
eventService.schedule(listOf(EventRequest("id_$it", time.plusSeconds(r.nextInt(60).toLong()).toString(), "default", "Payload_$it")))
}
(0..100).forEach { i ->
eventService.find("id_$i", "default").apply {
assertEquals(status, 200)
fetch<EventLookup> { it.xrefId = "id_$i"; it.tenant = "default" }.get()!!.apply {
assertEquals(shard, i / int("events.receiver.shard.size"))
fetch<Event> {
it.bucketId = time; it.shard = shard; it.eventTime = eventTime; it.id = eventId
}.get()!!.apply {
assertEquals(status, UN_PROCESSED)
}
}
}
}
}
@Test
fun `test bucket loader`() {
val bucketId = nowUTC().bucket()
val toBeLoaded = (1..10).map { bucketId.minusMinutes(it.toLong()) }.toSet()
save<Bucket> { it.bucketId = bucketId.minusMinutes(3); it.count = 100; it.status = PROCESSED }.get()!!
val latch = CountDownLatch(10)
val now = System.currentTimeMillis()
BucketsLoader(10, 5, 60, bucketId) {
try {
assertTrue { toBeLoaded.contains(it.bucketId) }
if (it.bucketId == bucketId.minusMinutes(3)) {
assertEquals(it.status, PROCESSED)
assertEquals(it.count, 100)
} else {
assertEquals(it.status, EMPTY)
}
latch.countDown()
} catch (e: Throwable) {
e.printStackTrace()
}
}.run()
if (!latch.await(1, MINUTES)) throw IllegalStateException("buckets registry did not complete on time")
assertTrue { System.currentTimeMillis() - now > 1 }
}
@Test
fun `test bucket manager`() {
val time = nowUTC().bucket()
println("time : $time")
val range = 0..9
val buckets = range.map { time.minusMinutes(it.toLong()) }.toSortedSet()
println("buckets: $buckets")
val shards = range.toList()
// test back ground load
range.forEach { i ->
save<Bucket> {
it.bucketId = time.minusMinutes(i.toLong()); it.count = 100L; it.status = UN_PROCESSED
}.get()!!
}
val bm = BucketManager(10, 2 * 60, 60, module())
bm.getProcessableShardsForOrBefore(time).get()!!
sleep(2000)
bm.getProcessableShardsForOrBefore(time).get()!!.apply {
assertEquals(this.keySet().toSortedSet(), buckets.toMutableSet().apply { add(time) }.toSortedSet())
this.keySet().forEach {
assertEquals(this[it].toList(), shards)
}
}
// test purge:
(1..5).forEach { i ->
save<Bucket> {
it.bucketId = time.plusMinutes(i.toLong()); it.count = 100L; it.status = UN_PROCESSED
}.get()!!
}
(1..5).forEach { bm.getProcessableShardsForOrBefore(time.plusMinutes(it.toLong())).get()!! }
bm.purgeIfNeeded()
(1..5).forEach {
val b = bm.getProcessableShardsForOrBefore(time.plusMinutes(1)).get()!!
assertEquals(b.keySet().size, 10)
assertEquals(buckets - buckets.take(5) + (1..5).map { time.plusMinutes(it.toLong()) }.toSortedSet(), b.keySet().toSortedSet())
}
}
@Test
fun `test http processor - ok case - past event`() {
var server: HttpServer? = null
try {
val port = 8383
eventService.registerProcessor(
ProcessorConfig(
"http", HTTP,
mapOf
(
"url" to "http://localhost:$port/test",
"headers" to mapOf("header" to "Header1")
)
)
).apply { assertEquals(this.status, 200) }
val eReq = EventRequest("id123", nowUTC().minusSeconds(1).toString(), "http", "Payload1")
println("event request: $eReq")
server = HttpServer.create(InetSocketAddress(port), 0)
val latch = CountDownLatch(1)
server.createContext("/test") {
try {
val eResp = EventResponse::class.java.fromJson(String(it.requestBody.readBytes()))
println("event response: $eResp")
assertEquals(it.requestHeaders.getFirst("header"), "Header1")
assertEquals(eReq.id, eResp.id)
assertEquals(eReq.eventTime, eResp.eventTime)
assertEquals(eReq.payload, eResp.payload)
assertEquals(eReq.tenant, eResp.tenant)
assertEquals(eReq.mode, eResp.mode)
assertTrue(eResp.eventId == null)
assertTrue(eResp.eventStatus == TRIGGERED)
mapOf("status" to "OK").json().apply {
it.sendResponseHeaders(200, length.toLong())
it.responseBody.write(toByteArray())
}
} catch (e: Throwable) {
e.printStackTrace()
mapOf("status" to "error").json().apply {
it.sendResponseHeaders(500, length.toLong())
it.responseBody.write(toByteArray())
}
throw AssertionError("test failed")
} finally {
it.close()
}
latch.countDown()
}
server.start()
eventService.schedule(listOf(eReq)).apply { assertEquals(200, status) }
if (!latch.await(1, MINUTES))
throw AssertionError("latch not down")
} finally {
server?.run { stop(0) }
}
}
@Test
fun `test http processor - ok case - future event`() {
var server: HttpServer? = null
try {
val port = 8383
eventService.registerProcessor(
ProcessorConfig(
"http", HTTP,
mapOf
(
"url" to "http://localhost:$port/test",
"headers" to mapOf("header" to "Header1")
)
)
).apply { assertEquals(this.status, 200) }
val time = nowUTC().plusMinutes(1).withSecond(10).withNano(0)
val eReq = EventRequest("id123", time.toString(), "http", "Payload1")
println("event request: $eReq")
server = HttpServer.create(InetSocketAddress(port), 0)
val latch = CountDownLatch(1)
server.createContext("/test") {
try {
val eResp = EventResponse::class.java.fromJson(String(it.requestBody.readBytes()))
println("event response: $eResp")
assertEquals(it.requestHeaders.getFirst("header"), "Header1")
assertEquals(eReq.id, eResp.id)
assertEquals(eReq.eventTime, eResp.eventTime)
assertEquals(eReq.payload, eResp.payload)
assertEquals("Payload2", eResp.payload)
assertEquals(eReq.tenant, eResp.tenant)
assertEquals(eReq.mode, eResp.mode)
assertTrue(eResp.eventId != null)
assertTrue(eResp.eventStatus == TRIGGERED)
mapOf("status" to "OK").json().apply {
it.sendResponseHeaders(200, length.toLong())
it.responseBody.write(toByteArray())
}
} catch (e: Throwable) {
e.printStackTrace()
mapOf("status" to "error").json().apply {
it.sendResponseHeaders(500, length.toLong())
it.responseBody.write(toByteArray())
}
throw AssertionError("test failed")
} finally {
it.close()
}
latch.countDown()
}
server.start()
eventService.schedule(listOf(eReq)).apply { assertEquals(200, status) }
eventService.schedule(listOf(eReq.apply { payload = "Payload2" })).apply { assertEquals(200, status) }
val bm = BucketManager(1, 2 * 60, 60, module())
println("manually scheduling $time")
ScheduleScanner(module()).scan(time.withSecond(0).withNano(0), bm)
if (!latch.await(2, MINUTES))
throw AssertionError("latch not down")
} finally {
server?.run { stop(0) }
}
}
@Test
fun `test http processor - error case`() {
var server: HttpServer? = null
try {
val port = 8383
eventService.registerProcessor(
ProcessorConfig(
"http", HTTP,
mapOf
(
"url" to "http://localhost:$port/test",
"header" to "Header1"
)
)
).apply { assertEquals(this.status, 200) }
val eReq = EventRequest("id123", nowUTC().minusSeconds(1).toString(), "http", "Payload1")
server = HttpServer.create(InetSocketAddress(port), 0)
val latch = CountDownLatch(1)
val tries = AtomicInteger()
server.createContext("/test") {
try {
tries.incrementAndGet()
mapOf("status" to "error").json().apply {
it.sendResponseHeaders(500, length.toLong())
it.responseBody.write(toByteArray())
}
} finally {
it.close()
}
latch.countDown()
}
server.start()
eventService.schedule(listOf(eReq)).apply { assertEquals(200, status) }
if (!latch.await(1, MINUTES))
throw AssertionError("latch not down")
var passed = false
loop@ for (i in (1..10)) {
if (tries.get() != 4)
sleep(1000)
else {
passed = true; break@loop
}
}
assertTrue(passed)
} finally {
server?.run { stop(0) }
}
}
@Test
fun `test http processor - bad request case`() {
var server: HttpServer? = null
try {
val port = 8383
eventService.registerProcessor(
ProcessorConfig(
"http", HTTP,
mapOf
(
"url" to "http://localhost:$port/test",
"header" to "Header1"
)
)
).apply { assertEquals(this.status, 200) }
val eReq = EventRequest("id123", nowUTC().minusSeconds(1).toString(), "http", "Payload1")
server = HttpServer.create(InetSocketAddress(port), 0)
val latch = CountDownLatch(1)
val tries = AtomicInteger()
server.createContext("/test") {
try {
tries.incrementAndGet()
mapOf("status" to "error").json().apply {
it.sendResponseHeaders(400, length.toLong())
it.responseBody.write(toByteArray())
}
} finally {
it.close()
}
latch.countDown()
}
server.start()
eventService.schedule(listOf(eReq)).apply { assertEquals(200, status) }
if (!latch.await(1, MINUTES))
throw AssertionError("latch not down")
var passed = false
loop@ for (i in (1..5)) {
if (tries.get() != 1)
sleep(1000)
else {
passed = true; break@loop
}
}
sleep(2000)
assertTrue(passed)
} finally {
server?.run { stop(0) }
}
}
@Test
fun `test kafka integration - ok case`() {
eventService.registerProcessor(
ProcessorConfig(
"kafka", MESSAGING,
mapOf
(
"topic" to "test",
"brokers.url" to ""
)
)
).apply { assertEquals(this.status, 200) }
val eReq = EventRequest("id123", nowUTC().minusSeconds(1).toString(), "kafka", "Payload1")
eventService.schedule(listOf(eReq)).apply { assertEquals(200, status) }
}
@Test
fun `test kafka integration - error case`() {
val x: ScheduledThreadPoolExecutor = (TaskExecutor.Companion::class.memberProperties
.filter { it.name == "RETRY_POOL" }[0]
.apply { isAccessible = true }.get(TaskExecutor.Companion) as ListeningScheduledExecutorService)
.let {
it::class.java.getDeclaredField("delegate").apply { isAccessible = true }.get(it)
} as ScheduledThreadPoolExecutor
val current = x.completedTaskCount
eventService.registerProcessor(
ProcessorConfig(
"kafka", MESSAGING,
mapOf
(
"topic" to "test",
"brokers.url" to "",
"fail" to true
)
)
).apply { assertEquals(this.status, 200) }
val eReq = EventRequest("id123", nowUTC().minusSeconds(1).toString(), "kafka", "Payload1")
eventService.schedule(listOf(eReq)).apply { assertEquals(200, status) }
sleep(10000)
assertTrue(x.completedTaskCount - current >= 3.toLong())
}
@Test(enabled = false)
fun `test kafka consumer`() {
eventService.registerProcessor(
ProcessorConfig(
"kafka", MESSAGING,
mapOf
(
"topic" to "outbound",
"brokers.url" to "localhost:9092"
)
)
).apply { assertEquals(this.status, 200) }
/*val consumer = (BigBen.messageProcessors[0] as MockKafkaProcessor).consumer
consumer.rebalance(setOf(TopicPartition("inbound", 1)))
consumer.updateBeginningOffsets(mapOf(TopicPartition("inbound", 0) to 1.toLong()))
consumer.updateBeginningOffsets(mapOf(TopicPartition("inbound", 0) to Long.MAX_VALUE))*/
val eReq = EventRequest("id123", nowUTC().minusSeconds(1).toString(), "kafka", "Payload1")
println(eReq.json())
//sleep(Long.MAX_VALUE)
}
@Test(enabled = false)
fun `end to end kafka`() {
eventService.registerProcessor(
ProcessorConfig(
"kafka", MESSAGING,
mapOf
(
"topic" to "outbound",
"bootstrap.servers" to "localhost:9092"
)
)
).apply { assertEquals(this.status, 200) }
val producer =
KafkaProducer<String, String>(map("kafka.producer.config").mapKeys { it.key.removePrefix("kafka.producer.config.") } +
mapOf
(
"topic" to "outbound",
"bootstrap.servers" to "localhost:9092"
))
(1..100).forEach {
println("sending $it")
val eReq =
EventRequest("id123", nowUTC().minusSeconds(1).toString(), "kafka", RandomStringGenerator.Builder().build().generate(1024))
producer.send(ProducerRecord("outbound", eReq.json())).get()
}
sleep(3000)
}
@Test(enabled = false)
fun `test consumer`() {
val consumer =
KafkaConsumer<String, String>(map("kafka.consumer.config") + mapOf("group.id" to UUID.randomUUID().toString()))
consumer.subscribe(setOf("outbound"))
while (true) {
println("polling outbound")
val records = consumer.poll(3000)
println(records.count())
consumer.commitSync()
}
}
}
================================================
FILE: app/src/test/kotlin/com/walmartlabs/bigben/tests/KafkaTests.kt
================================================
/*-
* #%L
* BigBen:commons
* =======================================
* Copyright (C) 2016 - 2018 Walmart Inc.
* =======================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.walmartlabs.bigben.tests
import com.datastax.driver.core.Session
import com.google.common.util.concurrent.Futures
import com.google.common.util.concurrent.ListenableFuture
import com.walmartlabs.bigben.BigBen
import com.walmartlabs.bigben.BigBen.entityProvider
import com.walmartlabs.bigben.BigBen.module
import com.walmartlabs.bigben.api.EventService
import com.walmartlabs.bigben.entities.EntityProvider
import com.walmartlabs.bigben.entities.Event
import com.walmartlabs.bigben.entities.EventRequest
import com.walmartlabs.bigben.entities.EventResponse
import com.walmartlabs.bigben.kafka.KafkaMessageProcessor
import com.walmartlabs.bigben.kafka.ProcessorImpl
import com.walmartlabs.bigben.processors.NoOpCustomClassProcessor
import com.walmartlabs.bigben.processors.ProcessorConfig
import com.walmartlabs.bigben.processors.ProcessorConfig.Type.CUSTOM_CLASS
import com.walmartlabs.bigben.processors.ProcessorConfig.Type.MESSAGING
import com.walmartlabs.bigben.processors.ProcessorRegistry
import com.walmartlabs.bigben.utils.commons.PropsLoader
import com.walmartlabs.bigben.utils.fromJson
import com.walmartlabs.bigben.utils.json
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.testng.annotations.BeforeMethod
import org.testng.annotations.Test
import java.time.ZonedDateTime
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
/**
* Created by smalik3 on 9/19/18
*/
class KafkaTests {
companion object {
val latch = CountDownLatch(1)
init {
System.setProperty("bigben.props", "file://bigben-kafka-test.yaml")
BigBen.init()
}
}
@BeforeMethod
private fun `clean up db`() {
println("cleaning up the db")
try {
(BigBen.module<EntityProvider<Any>>().unwrap() as Session).apply {
execute("truncate bigben.events;")
execute("truncate bigben.lookups;")
execute("truncate bigben.buckets;")
execute("truncate bigben.kv_table;")
}
} catch (e: Throwable) {
e.printStackTrace()
throw e
}
println("setting up tenant")
module<EventService>().registerProcessor(ProcessorConfig("tenant1", CUSTOM_CLASS,
mapOf("eventProcessorClass" to NoOpCustomClassProcessor::class.java.name)))
}
@Test(enabled = false)
fun `test kafka integration`() {
println("in kafka tests")
module<EventService>().registerProcessor(ProcessorConfig("tenant2", MESSAGING,
mapOf(
"topic" to "topic1",
"bootstrap.servers" to "localhost:9092"
)))
(1..1).forEach {
entityProvider<Event>().let { it.raw(it.selector(Event::class.java)) }.apply {
id = "id_$it"
eventTime = ZonedDateTime.parse("2018-09-19T20:42Z")
payload = "Payload_$it"
tenant = "tenant2"
xrefId = "xref_$id"
}.apply { module<ProcessorRegistry>().invoke(this).get() }
}
if (!latch.await(2, TimeUnit.MINUTES))
throw AssertionError("test failed")
}
}
class MockProcessorImpl(props: PropsLoader) : KafkaMessageProcessor(props) {
private val impl = ProcessorImpl(props)
private val counter = AtomicInteger(0)
init {
println("---> starting the kafka consumer")
println(EventRequest("id123", "2018-09-19T20:42Z", "tenant1", "Payload1").json())
}
override fun process(cr: ConsumerRecord<String, String>): ListenableFuture<Any> {
println("got a new record: ${cr.value()}")
val er = EventResponse::class.java.fromJson(cr.value())
val cr2 = ConsumerRecord(cr.topic(), cr.partition(), cr.offset(), cr.key(), er.run {
EventRequest(id, eventTime, tenant, payload).json()
})
impl.process(cr2).get()
counter.incrementAndGet()
if (counter.get() == 10) KafkaTests.latch.countDown()
return Futures.immediateFuture(cr)
}
}
================================================
FILE: app/src/test/resources/bigben-api-test.yaml
================================================
# top level modules
modules:
- name: domain
class: com.walmartlabs.bigben.providers.domain.cassandra.CassandraModule
- name: processors
object: com.walmartlabs.bigben.processors.ProcessorRegistry
- name: hz
class: com.walmartlabs.bigben.utils.hz.Hz
- name: scheduler
object: com.walmartlabs.bigben.SchedulerModule
- name: events
object: com.walmartlabs.bigben.EventModule
- name: messaging
object: com.walmartlabs.bigben.kafka.KafkaModule
enabled: false
- name: cron
object: com.walmartlabs.bigben.cron.CronRunner
enabled: false
# hazelcast properties
hz:
template: file://hz.template.xml
group:
name: bigben-dev
password: bigben-dev
network:
autoIncrementPort: true
members: 127.0.0.1
port: 5701
map:
store:
writeDelay: 30
# cassandra related properties
cassandra:
keyspace: bigben
cluster:
contactPoints: 127.0.0.1
clusterName: bigben-cluster
port: 9042
localDataCenter: null
coreConnectionsPerHost: 8
maxHostsPerConnection: 32768
keepTCPConnectionAlive: true
connectionTimeOut: 5000
readTimeout: 12000
reconnectPeriod: 5
username: null
password: null
downgradingConsistency: false
writeConsistency: "LOCAL_QUORUM"
readConsistency: "LOCAL_QUORUM"
# kafka related properties
kafka:
consumers:
- num.consumers: 8
processor.class: com.walmartlabs.bigben.kafka.ProcessorImpl
topics: null
max.poll.wait.time: 10000
message.retry.max.count: 10
unknown.exception.retries: 3
config:
key.deserializer: org.apache.kafka.common.serialization.StringDeserializer
value.deserializer: org.apache.kafka.common.serialization.StringDeserializer
bootstrap.servers: null
#fetch.min.bytes: 1
group.id: bigben-inbound
#heartbeat.interval.ms: 3000
session.timeout.ms: 30000
auto.offset.reset: earliest
fetch.max.bytes: 324000
max.poll.interval.ms: 30000
max.poll.records: 100
receive.buffer.bytes: 65536
request.timeout.ms: 60000
#send.buffer.bytes: 131072
enable.auto.commit: false
producer:
config: # this is default kafka producer config, these values will be used if not supplied during the tenant registration
key.serializer: org.apache.kafka.common.serialization.StringSerializer
value.serializer: org.apache.kafka.common.serialization.StringSerializer
acks: "1"
buffer.memory: 32400
retries: 3
# system properties
task:
executor:
#retry.thread.count: 8
retry.time.units: SECONDS
delay: 1
max.retries: 3
backoff.multiplier: 2
# scheduler / event related properties
events:
scheduler.enabled: true
schedule.scan.interval.minutes: 1
num.shard.submitters: 8
receiver:
shard.size: 1000
lapse.offset.minutes: 0
delete:
max.retries: 3
initial.delay: 1
backoff.multiplier: 1
submit:
initial.delay: 1
backoff.multiplier: 1
max.retries: 3
processor:
max.retries: 3
initial.delay: 1
backoff.multiplier: 2
eager.loading: true
tasks:
max.events.in.memory: 100000
scheduler.worker.threads: 8
# bucket manager / loader related properties
buckets:
backlog.check.limit: 300
background:
load.fetch.size: 100
load.wait.interval.seconds: 15
# cron related properties
cron:
runner:
core.pool.size: 8
load:
max.retries: 10
delay: 1
backoff.multiplier: 1
time.units: "SECONDS"
messaging.producer.factory.class: com.walmartlabs.bigben.kafka.MockMessageProducerFactory
generic.future.max.get.time: 60
================================================
FILE: app/src/test/resources/bigben-kafka-test.yaml
================================================
# top level modules
modules:
- name: domain
class: com.walmartlabs.bigben.providers.domain.cassandra.CassandraModule
- name: processors
object: com.walmartlabs.bigben.processors.ProcessorRegistry
- name: hz
class: com.walmartlabs.bigben.utils.hz.Hz
- name: scheduler
object: com.walmartlabs.bigben.SchedulerModule
- name: events
object: com.walmartlabs.bigben.EventModule
- name: kafka
object: com.walmartlabs.bigben.kafka.KafkaModule
# hazelcast properties
hz:
template: /hz.template.xml
group:
name: bigben-dev
password: bigben-dev
network:
autoIncrementPort: true
members: 127.0.0.1
port: 5701
map:
store:
writeDelay: 30
# cassandra related properties
cassandra:
keyspace: bigben
cluster:
contactPoints: 127.0.0.1
clusterName: bigben-cluster
port: 9042
localDataCenter: null
coreConnectionsPerHost: 8
maxHostsPerConnection: 32768
keepTCPConnectionAlive: true
connectionTimeOut: 5000
readTimeout: 12000
reconnectPeriod: 5
username: null
password: null
downgradingConsistency: false
writeConsistency: "LOCAL_QUORUM"
readConsistency: "LOCAL_QUORUM"
# system properties
task:
executor:
#retry.thread.count: 8
retry.time.units: SECONDS
delay: 1
max.retries: 3
backoff.multiplier: 2
# kafka related properties
kafka:
consumers:
- num.consumers: 1
processor.impl.class: com.walmartlabs.bigben.tests.MockProcessorImpl
topics: topic1
max.poll.wait.time: 10000
message.retry.max.count: 10
unknown.exception.retries: 3
config:
key.deserializer: org.apache.kafka.common.serialization.StringDeserializer
value.deserializer: org.apache.kafka.common.serialization.StringDeserializer
bootstrap.servers: localhost:9092
#fetch.min.bytes: 1
group.id: bigben-kafka-test
#heartbeat.interval.ms: 3000
session.timeout.ms: 30000
auto.offset.reset: earliest
fetch.max.bytes: 324000
max.poll.interval.ms: 30000
max.poll.records: 100
receive.buffer.bytes: 65536
request.timeout.ms: 60000
#send.buffer.bytes: 131072
enable.auto.commit: false
producer:
config: # this is default kafka producer config, these values will be used if not supplied during the tenant registration
key.serializer: org.apache.kafka.common.serialization.StringSerializer
value.serializer: org.apache.kafka.common.serialization.StringSerializer
acks: "1"
buffer.memory: 32400
retries: 3
messaging.producer.factory.class: com.walmartlabs.bigben.kafka.KafkaMessageProducerFactory
generic.future.max.get.time: 60
# scheduler / event related properties
events:
scheduler.enabled: false
schedule.scan.interval.minutes: 1
num.shard.submitters: 8
receiver:
shard.size: 10
lapse.offset.minutes: 0
delete:
max.retries: 3
initial.delay: 1
backoff.multiplier: 1
submit:
initial.delay: 1
backoff.multiplier: 1
max.retries: 3
processor:
max.retries: 3
initial.delay: 1
backoff.multiplier: 2
eager.loading: true
tasks:
max.events.in.memory: 100000
scheduler.worker.threads: 8
# bucket manager / loader related properties
buckets:
backlog.check.limit: 1
background:
load.fetch.size: 10
load.wait.interval.seconds: 1
checkpoint:
interval: 60
interval.units: SECONDS
================================================
FILE: app/src/test/resources/bigben-test.yaml
================================================
events:
receiver:
shard.size: 10
processor.eager.loading: false
================================================
FILE: app/src/test/resources/log4j.xml
================================================
<?xml version="1.0" encoding="UTF-8" ?>
<!--
#%L
BigBen:app
=======================================
Copyright (C) 2016 - 2018 Walmart Inc.
=======================================
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
#L%
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration debug="true"
xmlns:log4j='http://jakarta.apache.org/log4j/'
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://jakarta.apache.org/log4j/ ">
<appender name="console" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%t] %c{1}:%L - %m%n"/>
</layout>
</appender>
<logger name="com.walmartlabs.bigben" additivity="false">
<level value="INFO"/>
<appender-ref ref="console"/>
</logger>
</log4j:configuration>
================================================
FILE: build/configs/log4j.xml
================================================
<?xml version="1.0" encoding="UTF-8" ?>
<!--
#%L
BigBen:app
=======================================
Copyright (C) 2016 - 2018 Walmart Inc.
=======================================
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
#L%
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration debug="true"
xmlns:log4j='http://jakarta.apache.org/log4j/'
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://jakarta.apache.org/log4j/ ">
<appender name="console" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%t] %c{1}:%L - %m%n"/>
</layout>
</appender>
<appender name="file" class="org.apache.log4j.RollingFileAppender">
<param name="append" value="false"/>
<param name="maxFileSize" value="100MB"/>
<param name="maxBackupIndex" value="3"/>
<param name="file" value="${bigben.log.file}"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %-5p [%t] %c{1}:%L - %m%n"/>
</layout>
</appender>
<root>
<level value="INFO"/>
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</root>
<logger name="com.walmartlabs.bigben" additivity="false">
<level value="INFO"/>
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</logger>
</log4j:configuration>
================================================
FILE: build/configs/overrides.yaml
================================================
hz:
network.members: ${HZ_MEMBER_IPS}
cassandra.cluster.contactPoints: ${CASSANDRA_SEED_IPS}
bigben.inbound.topic:
name: some_topic
bootstrap.servers: some_servers
#events.scheduler.enabled: false
================================================
FILE: build/docker/Dockerfile
================================================
FROM openjdk:8-jre-alpine
ENV APPLICATION_USER bigben
RUN adduser -D -g '' $APPLICATION_USER
ENV APP_ROOT /dist
RUN if [ -d "$APP_ROOT" ]; then rm -Rf $APP_ROOT; fi
RUN mkdir $APP_ROOT
RUN chown -R $APPLICATION_USER $APP_ROOT
USER $APPLICATION_USER
COPY ./build/bin/bigben.jar $APP_ROOT/bigben.jar
COPY ./build/docker/start.sh $APP_ROOT/start.sh
USER root
RUN chmod +x $APP_ROOT/start.sh
WORKDIR $APP_ROOT
EXPOSE 8080 5701
CMD ["sh", "-c", "$APP_ROOT/start.sh"]
================================================
FILE: build/docker/app_run.sh
================================================
#!/usr/bin/env bash
set -e
APP_CONTAINER_NAME=${APP_CONTAINER_NAME:-bigben_app}
SERVER_PORT=${SERVER_PORT:-8080}
HZ_PORT=5701
NUM_INSTANCES=${NUM_INSTANCES:-1}
APP_ROOT=/dist
BUILD_DIR=${PWD}/..
LOGS_DIR=${LOGS_DIR:-${BUILD_DIR}/../../bigben_logs}
HOST_IP=${HOST_IP:-`ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'`}
CASSANDRA_SEED_IPS=${CASSANDRA_SEED_IPS:-${HOST_IP}}
HZ_MEMBER_IPS=${HZ_MEMBER_IPS:-${HOST_IP}}
DEFAULT_JAVA_OPTS="-server -XX:+UnlockExperimentalVMOptions -XX:InitialRAMFraction=2 -XX:MinRAMFraction=2 -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:+UseStringDeduplication"
JAVA_OPTS=${JAVA_OPTS}
if [[ "x${JAVA_OPTS}" != "x" ]]; then
JAVA_OPTS="${DEFAULT_JAVA_OPTS} ${JAVA_OPTS}"
else
JAVA_OPTS="${DEFAULT_JAVA_OPTS}"
fi
echo HOST_IP: ${HOST_IP}, SERVER_PORT: ${SERVER_PORT}, \
HZ_MEMBER_IPS: ${HZ_MEMBER_IPS}, CASSANDRA_SEED_IPS: ${CASSANDRA_SEED_IPS}, \
HZ_PORT: ${HZ_PORT}, NUM_INSTANCES: ${NUM_INSTANCES}
function stop() {
echo "stopping app servers, if any"
i=1
while [[ ${i} -lt $(($NUM_INSTANCES + 1)) ]]; do
app_port=$((${SERVER_PORT} + 101 * $((i - 1))))
echo "stopping ${APP_CONTAINER_NAME}_$app_port"
docker stop "${APP_CONTAINER_NAME}_$app_port" || true
let i=i+1
done
}
function start() {
echo "starting ${NUM_INSTANCES} app node(s)"
i=1
while [[ ${i} -lt $(($NUM_INSTANCES + 1)) ]]; do
app_port=$((${SERVER_PORT} + 101 * $((i - 1))))
hz_port=$((${HZ_PORT} + i - 1))
echo "starting ${APP_CONTAINER_NAME}_$app_port at app port: $app_port, hz port: $hz_port"
docker run -d --rm \
-p ${app_port}:${SERVER_PORT} \
-p ${hz_port}:${HZ_PORT} \
-v ${BUILD_DIR}/bin/bigben.yaml:${APP_ROOT}/bigben.yaml \
-v ${BUILD_DIR}/configs/overrides.yaml:${APP_ROOT}/overrides.yaml \
-v ${BUILD_DIR}/configs/log4j.xml:${APP_ROOT}/log4j.xml \
-v ${LOGS_DIR}:${APP_ROOT}/logs \
-e HOST_IP="${HOST_IP}" \
-e CASSANDRA_SEED_IPS="${CASSANDRA_SEED_IPS}" \
-e HZ_MEMBER_IPS="${HZ_MEMBER_IPS}" \
-e JAVA_OPTS="${JAVA_OPTS} -Dbigben.configs=uri://${APP_ROOT}/overrides.yaml,uri://${APP_ROOT}/bigben.yaml \
-Dapp.server.port=${SERVER_PORT} \
-Dbigben.log.file=${APP_ROOT}/logs/bigben_app_${app_port}.log \
-Dbigben.log.config=${APP_ROOT} \
-Dhazelcast.local.publicAddress=${HOST_IP}:${hz_port}" \
--name "${APP_CONTAINER_NAME}_$app_port" sandeepmalik/bigben:1
let i=i+1
done
echo "waiting for app servers to boot up"
i=1
while [[ ${i} -lt $(($NUM_INSTANCES + 1)) ]]; do
app_server_docker_ip=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${APP_CONTAINER_NAME}_$app_port"`
echo "waiting for app server ${APP_CONTAINER_NAME}_$app_port, docker ip: $app_server_docker_ip"
docker run --rm dadarek/wait-for-dependencies ${app_server_docker_ip}:${SERVER_PORT}
let i=i+1
done
}
if [[ $1 == "start" ]]; then
start
elif [[ $1 == "stop" ]]; then
stop
else
stop
start
fi
================================================
FILE: build/docker/cassandra_run.sh
================================================
#!/usr/bin/env bash
set -e
CASSANDRA_CONTAINER_NAME=${CASSANDRA_CONTAINER_NAME:-bigben_cassandra}
CASSANDRA_PORT=${CASSANDRA_PORT:-9042}
CASSANDRA_GOSSIP_PORT=${CASSANDRA_GOSSIP_PORT:-7000}
HOST_IP=${HOST_IP:-`ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'`}
echo "determined host ip: $HOST_IP"
echo "stopping ${CASSANDRA_CONTAINER_NAME}, if running"
docker stop ${CASSANDRA_CONTAINER_NAME} || true
echo "starting ${CASSANDRA_CONTAINER_NAME}"
docker run -d --rm \
-p ${CASSANDRA_PORT}:${CASSANDRA_PORT} \
-e CASSANDRA_BROADCAST_ADDRESS=${HOST_IP} \
-p ${CASSANDRA_GOSSIP_PORT}:${CASSANDRA_GOSSIP_PORT} \
-v ${PWD}/../bin/bigben-schema.cql:/tmp/bigben-schema.cql \
--name ${CASSANDRA_CONTAINER_NAME} cassandra
CASSANDRA_DOCKER_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${CASSANDRA_CONTAINER_NAME}`
echo "${CASSANDRA_CONTAINER_NAME} docker ip: ${CASSANDRA_DOCKER_IP}"
echo "waiting for ${CASSANDRA_CONTAINER_NAME} to boot up"
docker run --rm dadarek/wait-for-dependencies ${CASSANDRA_DOCKER_IP}:${CASSANDRA_PORT}
echo "creating bigben schema"
docker exec -it ${CASSANDRA_CONTAINER_NAME} cqlsh -f /tmp/bigben-schema.cql
================================================
FILE: build/docker/cleanup.sh
================================================
#!/usr/bin/env bash
docker stop $(docker ps -a -q)
docker rm $(docker ps -a -q)
================================================
FILE: build/docker/deploy.sh
================================================
#!/usr/bin/env bash
set -e
./docker_build.sh
docker push sandeepmalik/bigben:1
================================================
FILE: build/docker/docker-compose.yml
================================================
version: '3'
services:
cassandra:
image: cassandra:3
container_name: cassandra
hostname: cassandra
restart: on-failure
volumes:
- ../cassandra/src/main/resources/bigben-schema.cql:/tmp/bigben-schema.cql
bigben:
image: sandeepmalik/bigben:1
hostname: bigben
container_name: bigben
volumes:
- ../app/src/main/resources/bigben.yaml:/dist/bigben-config.yaml
- ./configs/overrides.yaml:/dist/bigben-overrides.yaml
- ./configs/log4j.xml:/dist/log4j-overrides.xml
- ./configs/hz.xml:/dist/hz.xml
setup_cassandra:
image: dadarek/wait-for-dependencies
container_name: setup_cassandra
depends_on:
- cassandra
command: cassandra:9042
================================================
FILE: build/docker/docker_build.sh
================================================
#!/usr/bin/env bash
set -e
../exec/build.sh
cd ../..
docker build -f build/docker/Dockerfile -t sandeepmalik/bigben:1 .
cd build/docker
================================================
FILE: build/docker/single_node_run.sh
================================================
#!/usr/bin/env bash
set -e
./cassandra_run.sh
export NUM_INSTANCES=1
./app_run.sh
================================================
FILE: build/docker/start.sh
================================================
#!/bin/sh
DEFAULT_JAVA_OPTS="-server -XX:+UnlockExperimentalVMOptions -XX:InitialRAMFraction=2 -XX:MinRAMFraction=2 -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:+UseStringDeduplication"
if [[ "x${JAVA_OPTS}" != "x" ]]; then export JAVA_OPTS="${DEFAULT_JAVA_OPTS} ${JAVA_OPTS}"; else export JAVA_OPTS="${DEFAULT_JAVA_OPTS}"; fi
echo "using JAVA_OPTS: ${JAVA_OPTS}"
java ${JAVA_OPTS} -jar bigben.jar
================================================
FILE: build/exec/app_run.sh
================================================
#!/usr/bin/env bash
export HOST_IP=${HOST_IP:-`ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'`}
export SERVER_PORT=${SERVER_PORT:-8080}
APP_ROOT=${PWD}/../configs
export HZ_MEMBER_IPS=${HZ_MEMBER_IPS:-${HOST_IP}}
export CASSANDRA_SEED_IPS=${CASSANDRA_SEED_IPS:-${HOST_IP}}
export LOGS_DIR=${LOGS_DIR:-${APP_ROOT}/../../../bigben_logs}
NUM_INSTANCES=${NUM_INSTANCES:-1}
HZ_PORT=${HZ_PORT:-5701}
echo HOST_IP: ${HOST_IP}, SERVER_PORT: ${SERVER_PORT}, \
HZ_MEMBER_IPS: ${HZ_MEMBER_IPS}, CASSANDRA_SEED_IPS: ${CASSANDRA_SEED_IPS}, \
HZ_PORT: ${HZ_PORT}, NUM_INSTANCES: ${NUM_INSTANCES}
DEFAULT_JAVA_OPTS="-server -XX:+UnlockExperimentalVMOptions -XX:InitialRAMFraction=2 -XX:MinRAMFraction=2 -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:+UseStringDeduplication"
JAVA_OPTS=${JAVA_OPTS}
echo "using JAVA_OPTS: ${JAVA_OPTS}"
if [[ "x${JAVA_OPTS}" != "x" ]]; then
export JAVA_OPTS="${DEFAULT_JAVA_OPTS} ${JAVA_OPTS}"
else
export JAVA_OPTS="${DEFAULT_JAVA_OPTS}"
fi
echo "starting ${NUM_INSTANCES} app node(s)"
i=1
while [[ ${i} -lt $(($NUM_INSTANCES + 1)) ]]; do
app_port=$((${SERVER_PORT} + 101 * $((i - 1))))
hz_port=$((${HZ_PORT} + i - 1))
echo "starting node $i at app port: $app_port, hz port: $hz_port, logs: ${LOGS_DIR}/bigben_app_${app_port}.log"
LOG_FILE="${LOGS_DIR}/bigben_app_${app_port}.log"
java ${JAVA_OPTS} \
-Dbigben.log.config=${APP_ROOT}/log4j.xml \
-Dbigben.log.file=${LOG_FILE} \
-Dapp.server.port=${app_port} \
-Dbigben.configs="uri://${APP_ROOT}/overrides.yaml,uri://${APP_ROOT}/../bin/bigben.yaml" \
-Dhz.network.port=${hz_port} \
-jar ../bin/bigben.jar > /dev/null &
if [[ ${NUM_INSTANCES} == 1 ]]; then
tail -f ${LOG_FILE}
fi
let i=i+1
done
================================================
FILE: build/exec/build.sh
================================================
#!/usr/bin/env bash
set -e
cd ../..
mvn clean install
rm -rf build/bin || true
mkdir build/bin
cp app/target/bigben.jar build/bin/
cp cassandra/src/main/resources/bigben-schema.cql ./build/bin/bigben-schema.cql
cp app/src/main/resources/bigben.yaml ./build/bin/bigben.yaml
================================================
FILE: build/exec/cleanup.sh
================================================
#!/usr/bin/env bash
set -e
ps aux | grep bigben.jar | grep -v grep | awk '{print $2}' | xargs kill -9
================================================
FILE: cassandra/LICENSE.txt
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2018 Sandeep Malik
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: cassandra/pom.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>bigben</artifactId>
<groupId>com.walmartlabs.bigben</groupId>
<version>1.0.7-SNAPSHOT</version>
</parent>
<artifactId>bigben-cassandra</artifactId>
<packaging>takari-jar</packaging>
<name>BigBen:cassandra</name>
<properties>
<cassandra-driver.version>3.2.0</cassandra-driver.version>
</properties>
<dependencies>
<dependency>
<groupId>com.walmartlabs.bigben</groupId>
<artifactId>bigben-lib</artifactId>
</dependency>
<dependency>
<groupId>com.datastax.cassandra</groupId>
<artifactId>cassandra-driver-core</artifactId>
<version>${cassandra-driver.version}</version>
</dependency>
<dependency>
<groupId>com.datastax.cassandra</groupId>
<artifactId>cassandra-driver-mapping</artifactId>
<version>${cassandra-driver.version}</version>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<buildDirectory>${project.build.directory}</buildDirectory>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>
</build>
</project>
================================================
FILE: cassandra/src/main/kotlin/com/walmartlabs/bigben/providers/domain/cassandra/CassandraModule.kt
================================================
/*-
* #%L
* BigBen:cassandra
* =======================================
* Copyright (C) 2016 - 2018 Walmart Inc.
* =======================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.walmartlabs.bigben.providers.domain.cassandra
import com.datastax.driver.core.*
import com.datastax.driver.core.HostDistance.LOCAL
import com.datastax.driver.core.HostDistance.REMOTE
import com.datastax.driver.core.policies.*
import com.datastax.driver.mapping.Mapper
import com.datastax.driver.mapping.Mapper.Option.consistencyLevel
import com.datastax.driver.mapping.Mapper.Option.saveNullFields
import com.datastax.driver.mapping.MappingManager
import com.google.common.util.concurrent.ListenableFuture
import com.walmartlabs.bigben.entities.*
import com.walmartlabs.bigben.extns.nowUTC
import com.walmartlabs.bigben.utils.commons.Module
import com.walmartlabs.bigben.utils.commons.ModuleRegistry
import com.walmartlabs.bigben.utils.commons.Props.map
import com.walmartlabs.bigben.utils.commons.Props.string
import com.walmartlabs.bigben.utils.fromJson
import com.walmartlabs.bigben.utils.json
import com.walmartlabs.bigben.utils.logger
import com.walmartlabs.bigben.utils.transform
import java.time.ZonedDateTime
/**
* Created by smalik3 on 3/2/18
*/
open class CassandraModule<T : Any> : EntityProvider<T>, ClusterFactory, EventLoader, Module {
companion object {
private val l = logger<CassandraModule<*>>()
private val cluster: Cluster
val mappingManager: MappingManager
private val loaderQuery: PreparedStatement
private val kvAllQuery: PreparedStatement
private val session: Session
private val clusterConfig = ClusterConfig::class.java.fromJson(map("cassandra.cluster").json())
private val writeConsistency = consistencyLevel(clusterConfig.writeConsistency)
private val readConsistency = consistencyLevel(clusterConfig.readConsistency)
init {
l.info("initialing the Cassandra module")
cluster = (Class.forName(string("domain.cluster.factory.class", CassandraModule::class.java.name)).newInstance() as ClusterFactory).create()
session = cluster.connect(string("cassandra.keyspace"))
mappingManager = MappingManager(session)
loaderQuery = mappingManager.session.prepare("SELECT * FROM ${session.loggedKeyspace}.events WHERE bucket_id = ? AND shard = ? AND (event_time, id) > (?,?) LIMIT ?;")
kvAllQuery = mappingManager.session.prepare("SELECT * FROM ${session.loggedKeyspace}.kv_table WHERE key = ?;")
}
}
override fun init(registry: ModuleRegistry) {
}
@Suppress("UNCHECKED_CAST")
override fun selector(type: Class<T>): T {
return when (type) {
Event::class.java -> EventC() as T
Bucket::class.java -> BucketC() as T
EventLookup::class.java -> EventLookupC() as T
KV::class.java -> KVC() as T
else -> throw IllegalArgumentException("unknown entity $type")
}
}
override fun raw(selector: T) = selector
override fun kvs(selector: KV): ListenableFuture<List<KV>> {
require(selector.key != null) { "key must be provided" }
return session.executeAsync(kvAllQuery.bind(selector.key)).transform {
it?.run { mappingManager.mapper(KVC::class.java).map(this).map { it } } ?: emptyList()
}
}
override fun fetch(selector: T): ListenableFuture<T?> {
return mappingManager.mapper(selector::class.java).let {
when (selector) {
is EventC -> {
require(
selector.eventTime != null && selector.id != null &&
selector.shard != null && selector.shard!! >= 0
) { "event keys not provided: $selector" }
it.getAsync(selector.bucketId, selector.shard, selector.eventTime, selector.id, readConsistency).transform { it }
}
is BucketC -> {
require(selector.bucketId != null) { "bucket id not provided: $selector" }
it.getAsync(selector.bucketId, readConsistency).transform { it }
}
is EventLookupC -> {
require(selector.tenant != null && selector.xrefId != null) { "look up keys not provided: $selector" }
it.getAsync(selector.tenant, selector.xrefId, readConsistency).transform { it }
}
is KVC -> {
require(selector.key != null && selector.column != null) { "kv keys not provided: $selector" }
it.getAsync(selector.key, selector.column, readConsistency).transform { it }
}
else -> throw IllegalArgumentException("unknown selector: $selector")
}
}.apply {
transform { if (l.isDebugEnabled) l.debug("fetched entity: {}", it) }
}
}
override fun save(selector: T): ListenableFuture<T> {
return mappingManager.mapper(selector::class.java).let {
@Suppress("UNCHECKED_CAST")
val m = it as Mapper<Any>
when (selector) {
is EventC -> {
require(
selector.eventTime != null && selector.id != null && selector.bucketId != null &&
selector.shard != null && selector.shard!! >= 0
) { "event keys not provided: $selector" }
}
is BucketC -> {
require(selector.bucketId != null) { "bucket id not provided: $selector" }
}
is EventLookupC -> {
require(selector.tenant != null && selector.xrefId != null) { "look up keys not provided: $selector" }
selector.lastModified = nowUTC()
}
is KVC -> {
require(selector.key != null && selector.column != null) { "kv keys not provided: $selector" }
selector.lastModified = nowUTC()
}
else -> throw IllegalArgumentException("unknown selector: $selector")
}
if (l.isDebugEnabled) l.debug("saving entity {}", selector)
m.saveAsync(selector, saveNullFields(false), writeConsistency).transform { _ -> if (l.isDebugEnabled) l.debug("saved entity {}", selector); selector }
}
}
override fun remove(selector: T): ListenableFuture<T> {
return mappingManager.mapper(selector::class.java).let {
@Suppress("UNCHECKED_CAST")
val m = it as Mapper<Any>
when (selector) {
is EventC -> {
require(
selector.eventTime != null && selector.id != null &&
selector.shard != null && selector.shard!! >= 0
) { "event keys not provided: $selector" }
}
is BucketC -> {
require(selector.bucketId != null) { "bucket id not provided: $selector" }
}
is EventLookupC -> {
require(selector.tenant != null && selector.xrefId != null) { "look up keys not provided: $selector" }
}
is KVC -> {
require(selector.key != null && selector.column != null) { "kv keys not provided: $selector" }
}
else -> throw IllegalArgumentException("unknown selector: $selector")
}
if (l.isDebugEnabled) l.debug("deleting entity: {}", selector)
m.deleteAsync(selector, writeConsistency).transform { _ -> if (l.isDebugEnabled) l.debug("deleted entity {}", selector); selector }
}
}
override fun create(): Cluster {
return Cluster.builder()
.withCodecRegistry(CodecRegistry().register(EnumCodec(EventStatus.values().toSet())).register(ZdtCodec()))
.withClusterName(clusterConfig.clusterName)
.withPort(clusterConfig.port)
.also { clusterConfig.compression?.run { it.withCompression(ProtocolOptions.Compression.valueOf(this)) } }
.withRetryPolicy(if (clusterConfig.downgradingConsistency) DowngradingConsistencyRetryPolicy.INSTANCE else DefaultRetryPolicy.INSTANCE)
.also {
clusterConfig.localDataCenter?.run {
it.withLoadBalancingPolicy(TokenAwarePolicy(DCAwareRoundRobinPolicy.builder().withLocalDc(this).withUsedHostsPerRemoteDc(0).build()))
}
}
.withReconnectionPolicy(ConstantReconnectionPolicy(clusterConfig.reconnectPeriod))
.withSocketOptions(SocketOptions().apply {
connectTimeoutMillis = clusterConfig.connectionTimeOut
readTimeoutMillis = clusterConfig.readTimeout
keepAlive = clusterConfig.keepTCPConnectionAlive
})
.withPoolingOptions(PoolingOptions().apply {
clusterConfig.apply {
setConnectionsPerHost(LOCAL, coreConnectionsPerLocalHost, maxConnectionsPerLocalHost)
setConnectionsPerHost(REMOTE, coreConnectionsPerRemoteHost, maxConnectionsPerRemoteHost)
}
heartbeatIntervalSeconds = 60
})
.also { clusterConfig.username?.run { it.withCredentials(this, clusterConfig.password) } }
.addContactPoints(*clusterConfig.contactPoints.split(",").toTypedArray())
.apply { decorate(this) }
.build()
}
protected open fun decorate(builder: Cluster.Builder) {
}
override fun unwrap() = session
override fun load(bucketId: ZonedDateTime, shard: Int, fetchSize: Int, eventTime: ZonedDateTime, eventId: String, context: Any?): ListenableFuture<Pair<Any?, List<Event>>> {
return mappingManager.session.executeAsync(loaderQuery.bind(bucketId, shard, eventTime, eventId, fetchSize)).transform { null to mappingManager.mapper(EventC::class.java).map(it!!).toList() }
}
}
================================================
FILE: cassandra/src/main/kotlin/com/walmartlabs/bigben/providers/domain/cassandra/ClusterConfig.kt
================================================
/*-
* #%L
* BigBen:cassandra
* =======================================
* Copyright (C) 2016 - 2018 Walmart Inc.
* =======================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.walmartlabs.bigben.providers.domain.cassandra
import com.datastax.driver.core.Cluster
import com.datastax.driver.core.ConsistencyLevel
import com.datastax.driver.core.ConsistencyLevel.LOCAL_ONE
import com.datastax.driver.core.SocketOptions.DEFAULT_CONNECT_TIMEOUT_MILLIS
import com.datastax.driver.core.SocketOptions.DEFAULT_READ_TIMEOUT_MILLIS
/**
* Created by smalik3 on 3/2/18
*/
data class ClusterConfig(
val clusterName: String = "bigben",
val contactPoints: String,
val port: Int = 9042,
val localDataCenter: String?,
val compression: String?,
val keepTCPConnectionAlive: Boolean = true,
val coreConnectionsPerLocalHost: Int = 1,
val maxConnectionsPerLocalHost: Int = 1,
val coreConnectionsPerRemoteHost: Int = 1,
val maxConnectionsPerRemoteHost: Int = 1,
val maxRequestsPerLocalConnection: Int = 32768,
val maxRequestsPerRemoteConnection: Int = 2048,
val newLocalConnectionThreshold: Int = 3000,
val newRemoteConnectionThreshold: Int = 400,
val poolTimeoutMillis: Int = 0,
val connectionTimeOut: Int = DEFAULT_CONNECT_TIMEOUT_MILLIS,
val readTimeout: Int = DEFAULT_READ_TIMEOUT_MILLIS,
val reconnectPeriod: Long = 5L,
val username: String?,
val password: String?,
val downgradingConsistency: Boolean = false,
val writeConsistency: ConsistencyLevel = LOCAL_ONE,
val readConsistency: ConsistencyLevel = LOCAL_ONE
)
interface ClusterFactory {
fun create(): Cluster
}
================================================
FILE: cassandra/src/main/kotlin/com/walmartlabs/bigben/providers/domain/cassandra/Entities.kt
================================================
/*-
* #%L
* BigBen:cassandra
* =======================================
* Copyright (C) 2016 - 2018 Walmart Inc.
* =======================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.walmartlabs.bigben.providers.domain.cassandra
import com.datastax.driver.mapping.annotations.*
import com.hazelcast.nio.ObjectDataInput
import com.hazelcast.nio.ObjectDataOutput
import com.walmartlabs.bigben.entities.*
import com.walmartlabs.bigben.extns.utc
import com.walmartlabs.bigben.hz.HzObjectFactory.Companion.BIGBEN_FACTORY_ID
import com.walmartlabs.bigben.hz.HzObjectFactory.ObjectId.BUCKET
import java.time.ZonedDateTime
import java.util.*
/**
* Created by smalik3 on 2/26/18
*/
@Table(name = "buckets")
data class BucketC(@PartitionKey @Column(name = "id") override var bucketId: ZonedDateTime? = null,
override var status: EventStatus? = null,
override var count: Long? = null,
@Column(name = "processed_at") override var processedAt: ZonedDateTime? = null,
@Column(name = "modified_at") override var updatedAt: ZonedDateTime? = null,
@Column(name = "failed_shards", codec = FailedShardsCodec::class) override var failedShards: Set<Int>? = null) : Bucket {
@Transient
override fun getFactoryId() = BIGBEN_FACTORY_ID
@Transient
override fun getId() = BUCKET.ordinal
override fun writeData(out: ObjectDataOutput) {
BitSet(4).apply {
set(0, bucketId != null)
set(1, status != null)
set(2, count != null)
set(3, processedAt != null)
set(4, updatedAt != null)
}.also { out.writeByteArray(it.toByteArray()) }.apply {
if (get(0)) out.writeLong(bucketId!!.toInstant().toEpochMilli())
if (get(1)) out.writeByte(status!!.ordinal)
if (get(2)) out.writeLong(count!!)
if (get(3)) out.writeLong(processedAt!!.toInstant().toEpochMilli())
if (get(4)) out.writeLong(updatedAt!!.toInstant().toEpochMilli())
}
}
override fun readData(ins: ObjectDataInput) {
BitSet.valueOf(ins.readByteArray()).apply {
if (get(0)) bucketId = utc(ins.readLong())
if (get(1)) status = EventStatus.values()[ins.readByte().toInt()]
if (get(2)) count = ins.readLong()
if (get(3)) processedAt = utc(ins.readLong())
if (get(4)) updatedAt = utc(ins.readLong())
}
}
}
@Table(name = "events")
data class EventC(@ClusteringColumn @Column(name = "event_time") override var eventTime: ZonedDateTime? = null,
@ClusteringColumn(1) override var id: String? = null,
@PartitionKey @Column(name = "bucket_id") override var bucketId: ZonedDateTime? = null,
@PartitionKey(1) override var shard: Int? = null,
override var status: EventStatus? = null,
override var error: String? = null,
override var tenant: String? = null,
@Column(name = "xref_id") override var xrefId: String? = null,
@Column(name = "processed_at") override var processedAt: ZonedDateTime? = null,
override var payload: String? = null,
@Transient override var eventResponse: EventResponse? = null,
@Transient override var deliveryOption: EventDeliveryOption? = null) : Event
@Table(name = "lookups")
data class EventLookupC(@PartitionKey override var tenant: String? = null,
@PartitionKey(1) @Column(name = "xref_id") override var xrefId: String? = null,
@Column(name = "bucket_id") override var bucketId: ZonedDateTime? = null,
override var shard: Int? = null,
@Column(name = "event_time") override var eventTime: ZonedDateTime? = null,
@Column(name = "event_id") override var eventId: String? = null,
override var payload: String? = null,
@Column(name = "l_m") var lastModified: ZonedDateTime? = null) : EventLookup
@Table(name = "kv_table")
data class KVC(@PartitionKey override var key: String? = null,
@ClusteringColumn override var column: String? = null,
override var value: String? = null,
@Column(name = "l_m") var lastModified: ZonedDateTime? = null
) : KV
================================================
FILE: cassandra/src/main/kotlin/com/walmartlabs/bigben/providers/domain/cassandra/codecs.kt
================================================
/*-
* #%L
* BigBen:cassandra
* =======================================
* Copyright (C) 2016 - 2018 Walmart Inc.
* =======================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.walmartlabs.bigben.providers.domain.cassandra
import com.datastax.driver.core.DataType
import com.datastax.driver.core.ProtocolVersion
import com.datastax.driver.core.TypeCodec
import com.google.common.reflect.TypeToken
import com.walmartlabs.bigben.extns.utc
import com.walmartlabs.bigben.utils.json
import com.walmartlabs.bigben.utils.typeRefJson
import java.nio.ByteBuffer
import java.time.ZonedDateTime
/**
* Created by smalik3 on 3/2/18
*/
class EnumCodec<T : Enum<T>>(values: Set<T>) : TypeCodec<T>(DataType.varchar(), @Suppress("UNCHECKED_CAST") (values.first()::class.java as Class<T>)) {
private val forward = values.associate { it.name to it }
override fun format(value: T) = value.name
override fun parse(value: String?) = value?.let { forward[it] }
override fun serialize(value: T?, protocolVersion: ProtocolVersion?) = value?.let { ByteBuffer.wrap(format(it).toByteArray()) }
override fun deserialize(bytes: ByteBuffer?, protocolVersion: ProtocolVersion?) = bytes?.let { parse(String(bytes.duplicate().array())) }
}
class ZdtCodec : TypeCodec<ZonedDateTime>(DataType.timestamp(), ZonedDateTime::class.java) {
override fun format(value: ZonedDateTime?) = value?.toInstant()?.toEpochMilli()?.toString()
override fun parse(value: String?): ZonedDateTime? = value?.let { utc(it.toLong()) }
override fun serialize(value: ZonedDateTime?, protocolVersion: ProtocolVersion?) = value?.let { ByteBuffer.allocate(8).apply { asLongBuffer().put(value.toInstant().toEpochMilli()) } }
override fun deserialize(bytes: ByteBuffer?, protocolVersion: ProtocolVersion?) = bytes?.let { utc(bytes.duplicate().asLongBuffer().get()) }
}
class FailedShardsCodec : TypeCodec<Set<Int>>(DataType.text(), object : TypeToken<Set<Int>>() {}) {
override fun format(value: Set<Int>?) = value?.json()
override fun parse(value: String?): Set<Int>? = value?.let { typeRefJson<Set<Int>>(it) }
override fun serialize(value: Set<Int>?, protocolVersion: ProtocolVersion?) = value?.let { ByteBuffer.wrap(it.json().toByteArray()) }
override fun deserialize(bytes: ByteBuffer?, protocolVersion: ProtocolVersion?) = bytes?.let { typeRefJson<Set<Int>>(String(it.array())) }
}
================================================
FILE: cassandra/src/main/resources/bigben-schema.cql
================================================
-- DROP KEYSPACE IF EXISTS bigben;
CREATE KEYSPACE IF NOT EXISTS bigben WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };
-- DROP TABLE IF EXISTS bigben.buckets;
CREATE TABLE IF NOT EXISTS bigben.buckets (
id timestamp PRIMARY KEY,
count bigint,
failed_shards text,
modified_at timestamp,
processed_at timestamp,
status text
);
-- DROP TABLE IF EXISTS bigben.lookups;
CREATE TABLE IF NOT EXISTS bigben.lookups (
tenant text,
xref_id text,
bucket_id timestamp,
event_id text,
event_time timestamp,
l_m timestamp,
payload text,
shard int,
PRIMARY KEY ((tenant, xref_id))
);
-- DROP TABLE IF EXISTS bigben.events;
CREATE TABLE IF NOT EXISTS bigben.events (
bucket_id timestamp,
shard int,
event_time timestamp,
id text,
error text,
payload text,
processed_at timestamp,
status text,
tenant text,
xref_id text,
PRIMARY KEY ((bucket_id, shard), event_time, id)
) WITH CLUSTERING ORDER BY (event_time ASC, id ASC);
-- DROP TABLE IF EXISTS bigben.kv_table;
CREATE TABLE IF NOT EXISTS bigben.kv_table (
key text,
column text,
l_m timestamp,
value text,
PRIMARY KEY (key, column)
) WITH CLUSTERING ORDER BY (column ASC);
================================================
FILE: cassandra/src/test/kotlin/com/walmartlabs/bigben/cassandra/tests/IntegrationTests.kt
================================================
/*-
* #%L
* BigBen:cassandra
* =======================================
* Copyright (C) 2016 - 2018 Walmart Inc.
* =======================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.walmartlabs.bigben.cassandra.tests
import com.datastax.driver.core.Session
import com.walmartlabs.bigben.BigBen
import com.walmartlabs.bigben.BigBen.module
import com.walmartlabs.bigben.entities.*
import com.walmartlabs.bigben.extns.bucket
import com.walmartlabs.bigben.extns.fetch
import com.walmartlabs.bigben.extns.nowUTC
import com.walmartlabs.bigben.extns.save
import org.testng.annotations.BeforeMethod
import org.testng.annotations.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
/**
* Created by smalik3 on 4/12/18
*/
class IntegrationTests {
companion object {
init {
System.setProperty("bigben.configs", "file://bigben-test.yaml")
BigBen.init()
}
}
@BeforeMethod
private fun `clean up db`() {
println("cleaning up the db")
try {
(module<EntityProvider<Any>>().unwrap() as Session).apply {
execute("truncate bigben.events;")
execute("truncate bigben.lookups;")
execute("truncate bigben.buckets;")
execute("truncate bigben.kv_table;")
}
} catch (e: Throwable) {
e.printStackTrace()
throw e
}
}
@Test
fun `test bucket`() {
val nowUTC = nowUTC().bucket()
save<Bucket> { it.bucketId = nowUTC; it.count = 10 }.get()
val bucket = fetch<Bucket> { it.bucketId = nowUTC }.get()!!
assertEquals(bucket.count, 10)
}
@Test
fun `test event loader`() {
val bucket = nowUTC().bucket()
val events = (0..99).map { i ->
save<Event> {
it.bucketId = bucket; it.shard = i / 10; it.eventTime = bucket.plusSeconds(10)
it.id = "e_$i"; it.status = EventStatus.UN_PROCESSED
}.get()
}.associate { "${it.eventTime}-${it.id}" to it }.toMutableMap()
val fetchSize = 20
(0..10).forEach {
var l = module<EventLoader>().load(bucket, it, fetchSize).get()
while (l.second.isNotEmpty()) {
l.second.forEach {
assertEquals(events["${it.eventTime}-${it.id}"], it)
events.remove("${it.eventTime}-${it.id}")
}
l =
module<EventLoader>().load(bucket, it, fetchSize, l.second.last().eventTime!!, l.second.last().id!!, l.first)
.get()
}
}
assertTrue { events.isEmpty() }
}
@Test
fun `event added successfully`() {
val bucket = nowUTC().bucket()
save<Event> {
it.bucketId = bucket; it.shard = 1; it.eventTime = bucket.plusSeconds(10)
it.id = "e1"; it.status = EventStatus.UN_PROCESSED
}.get()
val event = fetch<Event> {
it.bucketId = bucket; it.shard = 1; it.eventTime = bucket.plusSeconds(10); it.id = "e1"
}.get()!!
assertEquals(event.status, EventStatus.UN_PROCESSED)
}
}
================================================
FILE: cassandra/src/test/kotlin/com/walmartlabs/bigben/cassandra/tests/ORMTests.kt
================================================
/*-
* #%L
* BigBen:cassandra
* =======================================
* Copyright (C) 2016 - 2018 Walmart Inc.
* =======================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.walmartlabs.bigben.cassandra.tests
import com.walmartlabs.bigben.BigBen
import com.walmartlabs.bigben.entities.EventStatus.PROCESSED
import com.walmartlabs.bigben.entities.KV
import com.walmartlabs.bigben.extns.*
import com.walmartlabs.bigben.providers.domain.cassandra.BucketC
import com.walmartlabs.bigben.providers.domain.cassandra.CassandraModule.Companion.mappingManager
import com.walmartlabs.bigben.providers.domain.cassandra.EventC
import com.walmartlabs.bigben.providers.domain.cassandra.EventLookupC
import org.testng.annotations.Test
import java.util.*
import java.util.concurrent.TimeUnit.MINUTES
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
/**
* Created by smalik3 on 3/2/18
*/
class ORMTests {
companion object {
init {
System.setProperty("bigben.configs", "file://bigben-test.yaml")
BigBen.init()
}
}
@Test
fun `test bucket orm`() {
val b = BucketC(nowUTC(), PROCESSED, 10, nowUTC(), nowUTC())
val mapper = mappingManager.mapper(BucketC::class.java)
mapper.save(b)
val newBucket = mapper[b.bucketId]
assertEquals(b, newBucket)
}
@Test
fun `test event orm`() {
val eventTime = nowUTC()
val e = EventC(eventTime, UUID.randomUUID().toString(), eventTime.bucket(), 1, PROCESSED, null,
"default", processedAt = eventTime.plusSeconds(1), xrefId = "xref_1", payload = "{payload}")
val mapper = mappingManager.mapper(EventC::class.java)
mapper.save(e)
val newEventC = mapper[eventTime.bucket(), 1, eventTime, e.id]
assertEquals(e, newEventC)
}
@Test
fun `test event look up orm`() {
val el = EventLookupC("default", UUID.randomUUID().toString(), nowUTC().bucket(), 2, nowUTC(), "event_1", "payload1")
val mapper = mappingManager.mapper(EventLookupC::class.java)
mapper.save(el)
val newEventLookupC = mapper[el.tenant, el.xrefId]
assertEquals(el, newEventLookupC)
}
@Test
fun `test kv`() {
val key = UUID.randomUUID().toString()
save<KV> { it.key = key; it.column = 1.toString(); it.value = "Value1" }.get(1, MINUTES)
save<KV> { it.key = key; it.column = 2.toString(); it.value = "Value2" }.get(1, MINUTES)
val kv = fetch<KV> { it.key = key; it.column = 1.toString() }.get(1, MINUTES)
assertNotNull(kv)
assertEquals(kv.value, "Value1")
val kvs = kvs { it.key = key }.get(1, MINUTES)
assertEquals(kvs.size, 2)
kvs.associate { it.column to it.value }.apply {
assertEquals(this[1.toString()], "Value1")
assertEquals(this[2.toString()], "Value2")
}
}
}
================================================
FILE: cassandra/src/test/resources/bigben-test.yaml
================================================
# top level modules
modules:
- name: domain
class: com.walmartlabs.bigben.providers.domain.cassandra.CassandraModule
- name: processors
object: com.walmartlabs.bigben.processors.ProcessorRegistry
- name: hz
class: com.walmartlabs.bigben.utils.hz.Hz
- name: scheduler
object: com.walmartlabs.bigben.SchedulerModule
- name: events
object: com.walmartlabs.bigben.EventModule
# hazelcast properties
hz:
template: file://hz.template.xml
group:
name: bigben-dev
password: bigben-dev
network:
autoIncrementPort: true
members: 127.0.0.1
port: 5701
map:
store:
writeDelay: 30
# cassandra related properties
cassandra:
keyspace: bigben
cluster:
contactPoints: 127.0.0.1
clusterName: bigben-cluster
port: 9042
localDataCenter: null
coreConnectionsPerHost: 8
maxHostsPerConnection: 32768
keepTCPConnectionAlive: true
connectionTimeOut: 5000
readTimeout: 12000
reconnectPeriod: 5
username: null
password: null
downgradingConsistency: false
writeConsistency: "LOCAL_QUORUM"
readConsistency: "LOCAL_QUORUM"
# system properties
task:
executor:
#retry.thread.count: 8
retry.time.units: SECONDS
delay: 1
max.retries: 3
backoff.multiplier: 2
messaging.producer.factory.class: com.walmartlabs.bigben.processors.NoOpMessageProducerFactory
generic.future.max.get.time: 60
# scheduler / event related properties
events:
scheduler.enabled: true
schedule.scan.interval.minutes: 1
num.shard.submitters: 8
receiver:
shard.size: 10
lapse.offset.minutes: 0
delete:
max.retries: 3
initial.delay: 1
backoff.multiplier: 1
submit:
initial.delay: 1
backoff.multiplier: 1
max.retries: 3
processor:
max.retries: 3
initial.delay: 1
backoff.multiplier: 2
eager.loading: true
tasks:
max.events.in.memory: 100000
scheduler.worker.threads: 8
# bucket manager / loader related properties
buckets:
backlog.check.limit: 30
background:
load.fetch.size: 10
load.wait.interval.seconds: 1
checkpoint:
interval: 60
interval.units: SECONDS
# kafka related properties
kafka:
producer:
config: # this is default kafka producer config, these values will be used if not supplied during the tenant registration
key.serializer: org.apache.kafka.common.serialization.StringSerializer
value.serializer: org.apache.kafka.common.serialization.StringSerializer
acks: "1"
buffer.memory: 32400
retries: 3
================================================
FILE: cassandra/src/test/resources/log4j.xml
================================================
<?xml version="1.0" encoding="UTF-8" ?>
<!--
#%L
BigBen:app
=======================================
Copyright (C) 2016 - 2018 Walmart Inc.
=======================================
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
#L%
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration debug="true"
xmlns:log4j='http://jakarta.apache.org/log4j/'
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://jakarta.apache.org/log4j/ ">
<appender name="console" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern"
value="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%t] %c{1}:%L - %m%n"/>
</layout>
</appender>
<logger name="com.walmartlabs.bigben" additivity="false">
<level value="DEBUG"/>
<appender-ref ref="console"/>
</logger>
</log4j:configuration>
================================================
FILE: cassandra/src/test/resources/testng.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<!--
#%L
BigBen:cassandra
=======================================
Copyright (C) 2016 - 2018 Walmart Inc.
=======================================
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
#L%
-->
<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
<suite name="bigben-domain-tests">
<test name="domain-tests" verbose="1" parallel="false"/>
</suite>
================================================
FILE: commons/LICENSE.txt
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2018 Sandeep Malik
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: commons/pom.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>bigben</artifactId>
<groupId>com.walmartlabs.bigben</groupId>
<version>1.0.7-SNAPSHOT</version>
</parent>
<artifactId>bigben-commons</artifactId>
<packaging>takari-jar</packaging>
<name>BigBen:commons</name>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>io.github.microutils</groupId>
<artifactId>kotlin-logging</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-kotlin</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast</artifactId>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-reflect</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>
<version>2.9.5</version>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.25</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<buildDirectory>${project.build.directory}</buildDirectory>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>
</build>
</project>
================================================
FILE: commons/src/main/kotlin/com/walmartlabs/bigben/utils/_extns.kt
================================================
/*-
* #%L
* BigBen:commons
* =======================================
* Copyright (C) 2016 - 2018 Walmart Inc.
* =======================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.walmartlabs.bigben.utils
import com.fasterxml.jackson.core.JsonGenerator
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.core.Version
import com.fasterxml.jackson.core.type.TypeReference
import com.fasterxml.jackson.databind.*
import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.kotlin.KotlinModule
import com.google.common.base.Throwables
import mu.KotlinLogging
import org.slf4j.LoggerFactory
import java.time.ZonedDateTime
/**
* Created by smalik3 on 2/21/18
*/
inline fun <reified T : Any> logger() = KotlinLogging.logger(unwrapCompanionClass(T::class.java).name)
//LoggerFactory.getLogger(unwrapCompanionClass(T::class.java).name)!!
fun logger(name: String) = LoggerFactory.getLogger(name)!!
fun <T : Any> unwrapCompanionClass(ofClass: Class<T>): Class<*> {
return if (ofClass.enclosingClass != null && ofClass.enclosingClass.kotlin.isCompanion) {
ofClass.enclosingClass
} else {
ofClass
}
}
fun Throwable?.rootCause() = this?.let { Throwables.getRootCause(this) }
fun Throwable?.stackTraceAsString() = this?.let { Throwables.getStackTraceAsString(this) }
fun zdtModule() = SimpleModule("ZDT", Version(1, 0, 0, null, null, null)).also {
it.addSerializer(ZonedDateTime::class.java, object : JsonSerializer<ZonedDateTime>() {
override fun serialize(p0: ZonedDateTime?, p1: JsonGenerator, p2: SerializerProvider) {
p0?.let { p1.writeString(it.toString()) } ?: p1.writeNull()
}
})
it.addDeserializer(ZonedDateTime::class.java, object : JsonDeserializer<ZonedDateTime>() {
override fun deserialize(jp: JsonParser, dc: DeserializationContext): ZonedDateTime? {
return jp.codec.readValue(jp, String::class.java)?.let { ZonedDateTime.parse(it) }
}
})
}
typealias Json = Map<String, Any>
val om = ObjectMapper().registerModule(KotlinModule()).registerModule(zdtModule())!!
fun Any.json(): String = om.writeValueAsString(this)
fun Any.yaml(): String = omYaml.writeValueAsString(this)
fun <T> Class<T>.fromJson(s: String) = om.readValue(s, this)!!
fun <T> TypeReference<T>.fromJson(s: String): T = om.readValue(s, this)
inline fun <reified T> typeRefJson(s: String) = object : TypeReference<T>() {}.fromJson(s)
val omYaml = ObjectMapper(YAMLFactory()).registerModule(KotlinModule()).registerModule(zdtModule())!!
fun <T> Class<T>.fromYaml(s: String) = omYaml.readValue(s, this)!!
fun <T> TypeReference<T>.fromYaml(s: String): T = omYaml.readValue(s, this)
inline fun <reified T> typeRefYaml(s: String) = object : TypeReference<T>() {}.fromYaml(s)
================================================
FILE: commons/src/main/kotlin/com/walmartlabs/bigben/utils/_future_extns.kt
================================================
/*-
* #%L
* BigBen:commons
* =======================================
* Copyright (C) 2016 - 2018 Walmart Inc.
* =======================================
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.walmartlabs.bigben.utils
import com.google.common.base.Function
import com.google.common.base.Throwables.getStackTraceAsString
import com.google.common.util.concurrent.*
import com.google.common.util.concurrent.MoreExecutors.directExecutor
import com.hazelcast.core.ICompletableFuture
import com.walmartlabs.bigben.utils.commons.ListenableFutureAdapter
import com.walmartlabs.bigben.utils.commons.Props
import com.walmartlabs.bigben.utils.commons.TaskExecutor
import org.slf4j.LoggerFactory
import java.util.UUID.randomUUID
import java.util.concurrent.ExecutorService
import java.util.concurrent.Future
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeUnit.SECONDS
/**
* Created by smalik3 on 2/21/18
*/
val _l = LoggerFactory.getLogger("com.walmartlabs.bigben.utils")!!
fun onError(t: Throwable?) = _l.error("error in processing: ${getStackTraceAsString(t.rootCause()!!)}", t.rootCause())
fun <T> ListenableFuture<T>.done(onError: (Throwable?) -> Unit = ::onError, onSuccess: (T?) -> Unit): ListenableFuture<T> {
return also {
Futures.addCallback(it, object : FutureCallback<T> {
override fun onFailure(t: Throwable?) {
onError(t)
}
override fun onSuccess(result: T?) {
onSuccess(result)
}
}, directExecutor())
}
}
fun <T> List<ListenableFuture<T>>.done(onError: (Throwable?) -> Unit = ::onError, onSuccess: (List<T>?) -> Unit): ListenableFuture<List<T>> {
return reduce().done(onError, onSuccess)
}
fun <T> List<ListenableFuture<T>>.reduce(): ListenableFuture<List<T>> {
return Futures.allAsList(this)
}
fun <T, R> ListenableFuture<T>.transform(t: (T?) -> R): ListenableFuture<R> {
return Futures.transform(this, Function { t(it) }, directExecutor())
}
fun <T> ListenableFuture<T>.catching(t: (Throwable?) -> T): ListenableFuture<T> {
return Futures.catching(this, Exception::class.java, Function { t(it) }, directExecutor())
}
fun <T> ListenableFuture<T>.catchingAsync(t: (Throwable?) -> ListenableFuture<T>): ListenableFuture<T> {
return Futures.catchingAsync(this, Exception::class.java, AsyncFunction { t(it) }, directExecutor())
}
fun <T, R> ListenableFuture<T>.transformAsync(t: (T?) -> ListenableFuture<R>): ListenableFuture<R> {
return Futures.transformAsync(this, AsyncFunction { t(it) }, directExecutor())
}
fun <T> AsyncCallable<T>.scheduleAsync(delay: Long, units: TimeUnit, scheduledExecutor: ScheduledExecutorService): ListenableFuture<T> {
return Futures.scheduleAsync(this, delay, units, scheduledExecutor)
}
fun <T> AsyncCallable<T>.submitAsync(executorService: ExecutorService): ListenableFuture<T> {
return Futures.submitAsync(this, executorService)
}
private val te = TaskExecutor(setOf(Exception::class.java))
fun <T> (() -> ListenableFuture<T>).retriable(taskId: String = randomUUID().toString(),
maxRetries: Int = Props.int("task.executor.max.retries"),
delay: Int = Props.int("task.executor.delay"),
backoffMultiplier: Int = Props.int("task.executor.backoff.multiplier"),
gitextract_bc7hdl64/ ├── .gitignore ├── .looper.yml ├── LICENSE.txt ├── README.md ├── app/ │ ├── LICENSE.txt │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── kotlin/ │ │ │ └── com/ │ │ │ └── walmartlabs/ │ │ │ └── bigben/ │ │ │ └── app/ │ │ │ ├── app.kt │ │ │ └── run.kt │ │ └── resources/ │ │ ├── application.conf │ │ ├── bigben-lifecycle.yaml │ │ ├── bigben.yaml │ │ └── log4j.xml │ └── test/ │ ├── kotlin/ │ │ └── com/ │ │ └── walmartlabs/ │ │ └── bigben/ │ │ └── tests/ │ │ ├── APITests.kt │ │ ├── BigBenTests.kt │ │ └── KafkaTests.kt │ └── resources/ │ ├── bigben-api-test.yaml │ ├── bigben-kafka-test.yaml │ ├── bigben-test.yaml │ └── log4j.xml ├── build/ │ ├── configs/ │ │ ├── log4j.xml │ │ └── overrides.yaml │ ├── docker/ │ │ ├── Dockerfile │ │ ├── app_run.sh │ │ ├── cassandra_run.sh │ │ ├── cleanup.sh │ │ ├── deploy.sh │ │ ├── docker-compose.yml │ │ ├── docker_build.sh │ │ ├── single_node_run.sh │ │ └── start.sh │ └── exec/ │ ├── app_run.sh │ ├── build.sh │ └── cleanup.sh ├── cassandra/ │ ├── LICENSE.txt │ ├── pom.xml │ └── src/ │ ├── main/ │ │ ├── kotlin/ │ │ │ └── com/ │ │ │ └── walmartlabs/ │ │ │ └── bigben/ │ │ │ └── providers/ │ │ │ └── domain/ │ │ │ └── cassandra/ │ │ │ ├── CassandraModule.kt │ │ │ ├── ClusterConfig.kt │ │ │ ├── Entities.kt │ │ │ └── codecs.kt │ │ └── resources/ │ │ └── bigben-schema.cql │ └── test/ │ ├── kotlin/ │ │ └── com/ │ │ └── walmartlabs/ │ │ └── bigben/ │ │ └── cassandra/ │ │ └── tests/ │ │ ├── IntegrationTests.kt │ │ └── ORMTests.kt │ └── resources/ │ ├── bigben-test.yaml │ ├── log4j.xml │ └── testng.xml ├── commons/ │ ├── LICENSE.txt │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── kotlin/ │ │ └── com/ │ │ └── walmartlabs/ │ │ └── bigben/ │ │ └── utils/ │ │ ├── _extns.kt │ │ ├── _future_extns.kt │ │ ├── commons/ │ │ │ ├── ListenableFutureAdapter.kt │ │ │ ├── Props.kt │ │ │ ├── TaskExecutor.kt │ │ │ └── modules.kt │ │ └── hz/ │ │ ├── ClusterSingleton.kt │ │ ├── Hz.kt │ │ └── Service.kt │ └── test/ │ ├── kotlin/ │ │ └── PropsTests.kt │ └── resources/ │ ├── a.yaml │ ├── b.yaml │ ├── log4j.xml │ ├── overrides.yaml │ ├── props.yaml │ ├── sub1-overrides.yaml │ └── sub1.yaml ├── cron/ │ ├── pom.xml │ └── src/ │ └── main/ │ └── kotlin/ │ └── com/ │ └── walmartlabs/ │ └── bigben/ │ └── cron/ │ ├── cron-hz.kt │ ├── cron-processors.kt │ └── cron.kt ├── kafka/ │ ├── LICENSE.txt │ ├── pom.xml │ └── src/ │ └── main/ │ └── kotlin/ │ └── com/ │ └── walmartlabs/ │ └── bigben/ │ └── kafka/ │ ├── kafka-mocks.kt │ ├── kafka-module.kt │ ├── kafka-processor.kt │ └── kafka-producer.kt ├── lib/ │ ├── LICENSE.txt │ ├── pom.xml │ └── src/ │ └── main/ │ ├── kotlin/ │ │ └── com/ │ │ └── walmartlabs/ │ │ └── bigben/ │ │ ├── BigBen.kt │ │ ├── api/ │ │ │ ├── EventReceiver.kt │ │ │ └── EventService.kt │ │ ├── core/ │ │ │ ├── BucketManager.kt │ │ │ ├── BucketSnapshot.kt │ │ │ ├── BucketsLoader.kt │ │ │ ├── ScheduleScanner.kt │ │ │ └── StatusSyncer.kt │ │ ├── entities/ │ │ │ ├── EntityProvider.kt │ │ │ └── entities.kt │ │ ├── extns/ │ │ │ ├── _api_response_extns.kt │ │ │ ├── _bigben_extns.kt │ │ │ ├── _do_extns.kt │ │ │ └── _time_extns.kt │ │ ├── hz/ │ │ │ ├── BucketStore.kt │ │ │ └── HzObjectFactory.kt │ │ ├── modules.kt │ │ ├── processors/ │ │ │ ├── no_ops.kt │ │ │ └── processors.kt │ │ └── tasks/ │ │ └── tasks.kt │ └── resources/ │ └── hz.template.xml ├── pom.xml └── run_bigben_standalone.sh
Condensed preview — 99 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (435K chars).
[
{
"path": ".gitignore",
"chars": 473,
"preview": "**/target/\n\n# Mobile Tools for Java (J2ME)\n.mtj.tmp/\n\n# Package Files #\n*.ear\n\n# virtual machine crash logs, see http://"
},
{
"path": ".looper.yml",
"chars": 766,
"preview": "tools:\n jdk: 8\n maven: 3.5.2\n\ntriggers:\n - manual: Run default\n - manual:\n name: Release Build\n call: rele"
},
{
"path": "LICENSE.txt",
"chars": 11343,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README.md",
"chars": 14586,
"preview": "***\n# NOTICE:\n \n## This repository has been archived and is not supported.\n \n[![No Maintenance Intended](http://unmainta"
},
{
"path": "app/LICENSE.txt",
"chars": 11343,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "app/pom.xml",
"chars": 4488,
"preview": "<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocat"
},
{
"path": "app/src/main/kotlin/com/walmartlabs/bigben/app/app.kt",
"chars": 4565,
"preview": "/*-\n * #%L\n * BigBen:app\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "app/src/main/kotlin/com/walmartlabs/bigben/app/run.kt",
"chars": 4395,
"preview": "package com.walmartlabs.bigben.app\n\nimport com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT\nimport com."
},
{
"path": "app/src/main/resources/application.conf",
"chars": 298,
"preview": "ktor {\n deployment {\n port = 8080\n port = ${?app.server.port}\n }\n application {\n modules = [\n com.walma"
},
{
"path": "app/src/main/resources/bigben-lifecycle.yaml",
"chars": 42,
"preview": "pre-init-class: null\npost-init-class: null"
},
{
"path": "app/src/main/resources/bigben.yaml",
"chars": 4147,
"preview": "# top level modules\nmodules:\n - name: domain\n class: com.walmartlabs.bigben.providers.domain.cassandra.CassandraModu"
},
{
"path": "app/src/main/resources/log4j.xml",
"chars": 1603,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<!--\n #%L\n BigBen:app\n =======================================\n Copyright (C"
},
{
"path": "app/src/test/kotlin/com/walmartlabs/bigben/tests/APITests.kt",
"chars": 18563,
"preview": "package com.walmartlabs.bigben.tests\n\nimport com.datastax.driver.core.Session\nimport com.fasterxml.jackson.databind.Seri"
},
{
"path": "app/src/test/kotlin/com/walmartlabs/bigben/tests/BigBenTests.kt",
"chars": 22432,
"preview": "/*-\n * #%L\n * BigBen:app\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "app/src/test/kotlin/com/walmartlabs/bigben/tests/KafkaTests.kt",
"chars": 4875,
"preview": "/*-\n * #%L\n * BigBen:commons\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ==="
},
{
"path": "app/src/test/resources/bigben-api-test.yaml",
"chars": 3657,
"preview": "# top level modules\nmodules:\n - name: domain\n class: com.walmartlabs.bigben.providers.domain.cassandra.CassandraModu"
},
{
"path": "app/src/test/resources/bigben-kafka-test.yaml",
"chars": 3464,
"preview": "# top level modules\nmodules:\n - name: domain\n class: com.walmartlabs.bigben.providers.domain.cassandra.CassandraModu"
},
{
"path": "app/src/test/resources/bigben-test.yaml",
"chars": 71,
"preview": "events:\n receiver:\n shard.size: 10\n processor.eager.loading: false"
},
{
"path": "app/src/test/resources/log4j.xml",
"chars": 1511,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<!--\n #%L\n BigBen:app\n =======================================\n Copyright (C"
},
{
"path": "build/configs/log4j.xml",
"chars": 2120,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<!--\n #%L\n BigBen:app\n =======================================\n Copyright (C"
},
{
"path": "build/configs/overrides.yaml",
"chars": 202,
"preview": "hz:\n network.members: ${HZ_MEMBER_IPS}\ncassandra.cluster.contactPoints: ${CASSANDRA_SEED_IPS}\nbigben.inbound.topic:\n n"
},
{
"path": "build/docker/Dockerfile",
"chars": 463,
"preview": "FROM openjdk:8-jre-alpine\n\nENV APPLICATION_USER bigben\nRUN adduser -D -g '' $APPLICATION_USER\nENV APP_ROOT /dist\nRUN if "
},
{
"path": "build/docker/app_run.sh",
"chars": 3155,
"preview": "#!/usr/bin/env bash\nset -e\nAPP_CONTAINER_NAME=${APP_CONTAINER_NAME:-bigben_app}\nSERVER_PORT=${SERVER_PORT:-8080}\nHZ_PORT"
},
{
"path": "build/docker/cassandra_run.sh",
"chars": 1218,
"preview": "#!/usr/bin/env bash\nset -e\nCASSANDRA_CONTAINER_NAME=${CASSANDRA_CONTAINER_NAME:-bigben_cassandra}\nCASSANDRA_PORT=${CASSA"
},
{
"path": "build/docker/cleanup.sh",
"chars": 79,
"preview": "#!/usr/bin/env bash\ndocker stop $(docker ps -a -q)\ndocker rm $(docker ps -a -q)"
},
{
"path": "build/docker/deploy.sh",
"chars": 78,
"preview": "#!/usr/bin/env bash\nset -e\n./docker_build.sh\ndocker push sandeepmalik/bigben:1"
},
{
"path": "build/docker/docker-compose.yml",
"chars": 719,
"preview": "version: '3'\n\nservices:\n cassandra:\n image: cassandra:3\n container_name: cassandra\n hostname: cassandra\n re"
},
{
"path": "build/docker/docker_build.sh",
"chars": 135,
"preview": "#!/usr/bin/env bash\nset -e\n../exec/build.sh\ncd ../..\ndocker build -f build/docker/Dockerfile -t sandeepmalik/bigben:1 .\n"
},
{
"path": "build/docker/single_node_run.sh",
"chars": 81,
"preview": "#!/usr/bin/env bash\nset -e\n./cassandra_run.sh\nexport NUM_INSTANCES=1\n./app_run.sh"
},
{
"path": "build/docker/start.sh",
"chars": 394,
"preview": "#!/bin/sh\nDEFAULT_JAVA_OPTS=\"-server -XX:+UnlockExperimentalVMOptions -XX:InitialRAMFraction=2 -XX:MinRAMFraction=2 -XX:"
},
{
"path": "build/exec/app_run.sh",
"chars": 1821,
"preview": "#!/usr/bin/env bash\n\nexport HOST_IP=${HOST_IP:-`ifconfig | grep -Eo 'inet (addr:)?([0-9]*\\.){3}[0-9]*' | grep -Eo '([0-9"
},
{
"path": "build/exec/build.sh",
"chars": 273,
"preview": "#!/usr/bin/env bash\nset -e\ncd ../..\nmvn clean install\nrm -rf build/bin || true\nmkdir build/bin\ncp app/target/bigben.jar "
},
{
"path": "build/exec/cleanup.sh",
"chars": 101,
"preview": "#!/usr/bin/env bash\nset -e\nps aux | grep bigben.jar | grep -v grep | awk '{print $2}' | xargs kill -9"
},
{
"path": "cassandra/LICENSE.txt",
"chars": 11343,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "cassandra/pom.xml",
"chars": 2084,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "cassandra/src/main/kotlin/com/walmartlabs/bigben/providers/domain/cassandra/CassandraModule.kt",
"chars": 10686,
"preview": "/*-\n * #%L\n * BigBen:cassandra\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ="
},
{
"path": "cassandra/src/main/kotlin/com/walmartlabs/bigben/providers/domain/cassandra/ClusterConfig.kt",
"chars": 2217,
"preview": "/*-\n * #%L\n * BigBen:cassandra\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ="
},
{
"path": "cassandra/src/main/kotlin/com/walmartlabs/bigben/providers/domain/cassandra/Entities.kt",
"chars": 4999,
"preview": "/*-\n * #%L\n * BigBen:cassandra\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ="
},
{
"path": "cassandra/src/main/kotlin/com/walmartlabs/bigben/providers/domain/cassandra/codecs.kt",
"chars": 2946,
"preview": "/*-\n * #%L\n * BigBen:cassandra\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ="
},
{
"path": "cassandra/src/main/resources/bigben-schema.cql",
"chars": 1180,
"preview": "-- DROP KEYSPACE IF EXISTS bigben;\n\nCREATE KEYSPACE IF NOT EXISTS bigben WITH REPLICATION = { 'class' : 'SimpleStrategy'"
},
{
"path": "cassandra/src/test/kotlin/com/walmartlabs/bigben/cassandra/tests/IntegrationTests.kt",
"chars": 3741,
"preview": "/*-\n * #%L\n * BigBen:cassandra\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ="
},
{
"path": "cassandra/src/test/kotlin/com/walmartlabs/bigben/cassandra/tests/ORMTests.kt",
"chars": 3469,
"preview": "/*-\n * #%L\n * BigBen:cassandra\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ="
},
{
"path": "cassandra/src/test/resources/bigben-test.yaml",
"chars": 2541,
"preview": "# top level modules\nmodules:\n - name: domain\n class: com.walmartlabs.bigben.providers.domain.cassandra.CassandraModu"
},
{
"path": "cassandra/src/test/resources/log4j.xml",
"chars": 1512,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<!--\n #%L\n BigBen:app\n =======================================\n Copyright (C"
},
{
"path": "cassandra/src/test/resources/testng.xml",
"chars": 918,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n #%L\n BigBen:cassandra\n =======================================\n Copyrig"
},
{
"path": "commons/LICENSE.txt",
"chars": 11343,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "commons/pom.xml",
"chars": 2964,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "commons/src/main/kotlin/com/walmartlabs/bigben/utils/_extns.kt",
"chars": 3418,
"preview": "/*-\n * #%L\n * BigBen:commons\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ==="
},
{
"path": "commons/src/main/kotlin/com/walmartlabs/bigben/utils/_future_extns.kt",
"chars": 4771,
"preview": "/*-\n * #%L\n * BigBen:commons\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ==="
},
{
"path": "commons/src/main/kotlin/com/walmartlabs/bigben/utils/commons/ListenableFutureAdapter.kt",
"chars": 2354,
"preview": "/*-\n * #%L\n * BigBen:commons\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ==="
},
{
"path": "commons/src/main/kotlin/com/walmartlabs/bigben/utils/commons/Props.kt",
"chars": 9012,
"preview": "/*-\n * #%L\n * BigBen:commons\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ==="
},
{
"path": "commons/src/main/kotlin/com/walmartlabs/bigben/utils/commons/TaskExecutor.kt",
"chars": 5858,
"preview": "/*-\n * #%L\n * BigBen:commons\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ==="
},
{
"path": "commons/src/main/kotlin/com/walmartlabs/bigben/utils/commons/modules.kt",
"chars": 2762,
"preview": "/*-\n * #%L\n * BigBen:commons\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ==="
},
{
"path": "commons/src/main/kotlin/com/walmartlabs/bigben/utils/hz/ClusterSingleton.kt",
"chars": 4170,
"preview": "/*-\n * #%L\n * BigBen:commons\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ==="
},
{
"path": "commons/src/main/kotlin/com/walmartlabs/bigben/utils/hz/Hz.kt",
"chars": 2907,
"preview": "/*-\n * #%L\n * BigBen:commons\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ==="
},
{
"path": "commons/src/main/kotlin/com/walmartlabs/bigben/utils/hz/Service.kt",
"chars": 902,
"preview": "/*-\n * #%L\n * BigBen:commons\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ==="
},
{
"path": "commons/src/test/kotlin/PropsTests.kt",
"chars": 6920,
"preview": "/*-\n * #%L\n * BigBen:commons\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ==="
},
{
"path": "commons/src/test/resources/a.yaml",
"chars": 3656,
"preview": "# top level modules\nmodules:\n - name: domain\n class: com.walmartlabs.bigben.providers.domain.cassandra.CassandraModu"
},
{
"path": "commons/src/test/resources/b.yaml",
"chars": 66,
"preview": "inbound.topics.1: my_topic\ninbound.bootstrap.servers.1: my_servers"
},
{
"path": "commons/src/test/resources/log4j.xml",
"chars": 1681,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<!--\n #%L\n BigBen:commons\n =======================================\n Copyrigh"
},
{
"path": "commons/src/test/resources/overrides.yaml",
"chars": 101,
"preview": "---\na:\n c.d: y1\n g:\n - 1\n - 3\n h:\n h1: ${user.homE:-abc}\n h3: ${user.home:-abc}\n j: 1"
},
{
"path": "commons/src/test/resources/props.yaml",
"chars": 88,
"preview": "---\na:\n b: x\n c.d: y\n e: 12\n f: true\n g:\n - 1\n - 2\n h:\n h1: H1\n h2: H2"
},
{
"path": "commons/src/test/resources/sub1-overrides.yaml",
"chars": 237,
"preview": "a: b\nc:\n - d:\n d11: ${java.home1:-acc}\n d22: D22\n e: E22\n l:\n - a: ${java.io.tmpdir1:-Aaa}\n "
},
{
"path": "commons/src/test/resources/sub1.yaml",
"chars": 122,
"preview": "a: b\nc:\n - d:\n e: E1\n l:\n - a: b\n - a1: b1\n - g: h\n - G: H\n - i:\n j: k\n l: m\n - 4"
},
{
"path": "cron/pom.xml",
"chars": 1338,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "cron/src/main/kotlin/com/walmartlabs/bigben/cron/cron-hz.kt",
"chars": 3402,
"preview": "/*-\n * #%L\n * Bigben:cron\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======"
},
{
"path": "cron/src/main/kotlin/com/walmartlabs/bigben/cron/cron-processors.kt",
"chars": 3795,
"preview": "/*-\n * #%L\n * Bigben:cron\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======"
},
{
"path": "cron/src/main/kotlin/com/walmartlabs/bigben/cron/cron.kt",
"chars": 9960,
"preview": "/*-\n * #%L\n * Bigben:cron\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======"
},
{
"path": "kafka/LICENSE.txt",
"chars": 11343,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "kafka/pom.xml",
"chars": 1151,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "kafka/src/main/kotlin/com/walmartlabs/bigben/kafka/kafka-mocks.kt",
"chars": 2619,
"preview": "/*-\n * #%L\n * bigben-kafka\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ====="
},
{
"path": "kafka/src/main/kotlin/com/walmartlabs/bigben/kafka/kafka-module.kt",
"chars": 11602,
"preview": "/*-\n * #%L\n * bigben-kafka\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ====="
},
{
"path": "kafka/src/main/kotlin/com/walmartlabs/bigben/kafka/kafka-processor.kt",
"chars": 3172,
"preview": "/*-\n * #%L\n * bigben-kafka\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ====="
},
{
"path": "kafka/src/main/kotlin/com/walmartlabs/bigben/kafka/kafka-producer.kt",
"chars": 3320,
"preview": "/*-\n * #%L\n * bigben-kafka\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ====="
},
{
"path": "lib/LICENSE.txt",
"chars": 11343,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "lib/pom.xml",
"chars": 1255,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/BigBen.kt",
"chars": 2296,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/api/EventReceiver.kt",
"chars": 12078,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/api/EventService.kt",
"chars": 5509,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/core/BucketManager.kt",
"chars": 9793,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/core/BucketSnapshot.kt",
"chars": 2736,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/core/BucketsLoader.kt",
"chars": 3814,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/core/ScheduleScanner.kt",
"chars": 10028,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/core/StatusSyncer.kt",
"chars": 2796,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/entities/EntityProvider.kt",
"chars": 1470,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/entities/entities.kt",
"chars": 4972,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/extns/_api_response_extns.kt",
"chars": 3054,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/extns/_bigben_extns.kt",
"chars": 2603,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/extns/_do_extns.kt",
"chars": 1623,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/extns/_time_extns.kt",
"chars": 2532,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/hz/BucketStore.kt",
"chars": 2568,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/hz/HzObjectFactory.kt",
"chars": 2251,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/modules.kt",
"chars": 2185,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/processors/no_ops.kt",
"chars": 1581,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/processors/processors.kt",
"chars": 12395,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/kotlin/com/walmartlabs/bigben/tasks/tasks.kt",
"chars": 9704,
"preview": "/*-\n * #%L\n * BigBen:lib\n * =======================================\n * Copyright (C) 2016 - 2018 Walmart Inc.\n * ======="
},
{
"path": "lib/src/main/resources/hz.template.xml",
"chars": 3628,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n #%L\n BigBen:lib\n =======================================\n Copyright (C)"
},
{
"path": "pom.xml",
"chars": 16629,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns=\"http://maven.apache.org/POM/4.0.0\" xmlns:xsi=\"http://www.w3.org/2"
},
{
"path": "run_bigben_standalone.sh",
"chars": 114,
"preview": "#!/usr/bin/env bash\necho Building BigBen\nmvn clean install\ncd app/target\necho Starting BigBen\njava -jar bigben.jar"
}
]
About this extraction
This page contains the full source code of the walmartlabs/bigben GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 99 files (405.4 KB), approximately 96.7k tokens. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.