[ { "path": ".gitignore", "content": "### Gradle template\n.gradle\nbuild/\n\n# Ignore Gradle GUI config\ngradle-app.setting\n\n# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored)\n!gradle-wrapper.jar\n### JetBrains template\n# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio\n\n*.iml\n\n## Directory-based project format:\n.idea/\n# if you remove the above rule, at least ignore the following:\n\n# User-specific stuff:\n# .idea/workspace.xml\n# .idea/tasks.xml\n# .idea/dictionaries\n\n# Sensitive or high-churn files:\n# .idea/dataSources.ids\n# .idea/dataSources.xml\n# .idea/sqlDataSources.xml\n# .idea/dynamic.xml\n# .idea/uiDesigner.xml\n\n# Gradle:\n# .idea/gradle.xml\n# .idea/libraries\n\n# Mongo Explorer plugin:\n# .idea/mongoSettings.xml\n\n## File-based project format:\n*.ipr\n*.iws\n\n## Plugin-specific files:\n\n# IntelliJ\n/out/\n\n# mpeltonen/sbt-idea plugin\n.idea_modules/\n\n# JIRA plugin\natlassian-ide-plugin.xml\n\n# Crashlytics plugin (for Android Studio and IntelliJ)\ncom_crashlytics_export_strings.xml\ncrashlytics.properties\ncrashlytics-build.properties\n\n# Created by .ignore support plugin (hsz.mobi)\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright {yyyy} {name of copyright owner}\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "# kafka-streams\nThis is the repository for the examples of using Kafka streams covered in the blog posts: \n\n * [Kafka Streams - The Processor API](http://codingjunkie.net/kafka-processor-part1/)\n * [Kafka Streams - The KStreams API](http://codingjunkie.net/kafka-streams-part2/)\n * [Machine Learning with Kafka Streams](http://codingjunkie.net/kafka-streams-machine-learning/)\n\n\n## Requirements to build this project\n\n1. Java 8\n2. Gradle\n\n## Requirements to run the examples\n\n1. [kafka](https://github.com/apache/kafka) version kafka_2.11-0.10.1.0 see the section marked \"Running a task on a particular version of Scala\"\n2. The [json-data-generator](https://github.com/acesinc/json-data-generator) from [ACES,Inc](http://acesinc.net/) \n\n\n## Setup Instructions\n\n#### Extact the kafka_2.11-0.10.1.0.tgz file ####\n tar -xvzf kafka_2.11-0.10.1.0.tgz\n\n\n#### Start zookeeper and kafka\n```\n kafka-install-dir/bin/zookeeper-server-start.sh kafka-install-dir/conf/zookeeper.properties\n kafka-install-dir/bin/kafka-server-start.sh kafka-install-dir/conf/server.properties\n```\n\n#### Install the Json-Data-Generator \nDownload the latest [json-data-generator release](https://github.com/acesinc/json-data-generator/releases) and follow the install instructions [here](http://acesinc.net/introducing-a-streaming-json-data-generator/)\n\n#### Setup the kafka-streams repo\nClone or fork the repo\n```\n git clone git@github.com:bbejeck/kafka-streams \n cd kafka-streams\n``` \nThen copy the json config files to json generator conf directory\n```\n cp streaming-workflows/* /json-data-generator-1.2.0/conf\n``` \n \nCreate all the topics required by the examples\n```\n ./bin/create-topics.sh /usr/local/kafka_2.11-0.10.1.0 localhost 2181\n args are kafka home, zookeeper host and zookeeper port adjust accordingly\n``` \n\n### Running the Purchase Processor API KStreams API Examples ###\n cd /json-data-generator-1.2.0/\n java -jar json-data-generator-1.2.0 purchases-config.json\n cd kafka-streams\n ./gradlew runPurchaseProcessor | runPurchaseStreams\n \n\n### Running the Stock Trades Processor API or KStreams API Examples ###\n cd /json-data-generator-1.2.0/\n java -jar json-data-generator-1.2.0 stock-transactions-config.json\n cd kafka-streams\n ./gradlew runStockProcessor | runStockStreams\n \n### Running the Twitter KStreams Language Classification Example ###\n rename src/main/resources/twitter-app.properties.template to twitter-app.properties \n fill out the properties file with all the required values\n \n cd kafka-streams\n ./gradlew runTwitterKstreamNLP \n\n### Viewing the results of the purchase streaming examples ###\n cd kafka_install-dir/bin\n ./kafka-console-consumer --topic [patterns|rewards|purchases] --zookeeper localhost:2181\n \n### Viewing the results of the stock-trading streaming examples ###\n cd kafka_install-dir/bin\n ./kafka-console-consumer --topic [stocks-out|transaction-summary] --zookeeper localhost:2181\n \n### Viewing the results of the Twitter KStreams Language Classification Example ###\n cd kafka_install-dir/bin\n ./kafka-console-consumer --topic [english|french|spanish] --zookeeper localhost:2181 \n \n" }, { "path": "bin/create-topics.sh", "content": "#!/usr/bin/env bash\n\n\nKAFKA_HOME=$1\nZK_HOST=$2\nZK_PORT=$3\n\ntopics=\"src-topic patterns rewards purchases stocks stocks-out transaction-summary twitterData english french spanish\"\n\nfor topic in ${topics}; do\n echo \"attempting to create topic ${topic}\"\n ${KAFKA_HOME}/bin/kafka-topics.sh --create --topic ${topic} --partitions 1 --replication-factor 1 --zookeeper ${ZK_HOST}:${ZK_PORT}\ndone\n" }, { "path": "build.gradle", "content": "apply plugin: 'java'\napply plugin: 'maven'\napply plugin: 'idea'\napply plugin: 'application'\n\ngroup = 'bbejeck'\nversion = '1.0-SNAPSHOT'\n\ndescription = \"\"\"\"\"\"\n\nsourceCompatibility = 1.8\ntargetCompatibility = 1.8\n\n\ntask runPurchaseProcessor(type: JavaExec){\n classpath sourceSets.main.runtimeClasspath\n main = \"bbejeck.processor.purchases.PurchaseProcessorDriver\"\n}\n\ntask runPurchaseStreams(type: JavaExec){\n classpath sourceSets.main.runtimeClasspath\n main = \"bbejeck.streams.purchases.PurchaseKafkaStreamsDriver\"\n}\n\ntask runStockProcessor(type: JavaExec){\n classpath sourceSets.main.runtimeClasspath\n main = \"bbejeck.processor.stocks.StockSummaryStatefulProcessorDriver\"\n}\n\ntask runStockStreams(type: JavaExec){\n classpath sourceSets.main.runtimeClasspath\n main = \"bbejeck.streams.stocks.StocksKafkaStreamsDriver\"\n}\n\ntask runTwitterKstreamNLP(type: JavaExec) {\n classpath sourceSets.main.runtimeClasspath\n main = \"bbejeck.streams.twitter.TwitterKStreamNLPDriver\"\n}\n\n\n\n\njar {\n archiveName='bbejeck-kafka-streams-1.0-SNAPSHOT.jar'\n}\n\nrepositories {\n mavenCentral()\n}\n\ndependencies {\n compile group: 'com.google.code.gson', name: 'gson', version:'2.6.1'\n compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version:'2.7.1'\n compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version:'2.7.1'\n compile group: 'com.google.guava', name: 'guava', version:'19.0'\n compile group: 'org.apache.commons', name: 'commons-lang3', version:'3.1'\n compile group: 'com.101tec', name:'zkclient', version:'0.7'\n compile group: 'com.yammer.metrics', name:'metrics-core', version:'2.2.0'\n compile group: 'com.twitter', name:'hbc-core', version:'2.2.0'\n compile group: 'de.julielab', name: 'aliasi-lingpipe', version:'4.1.0'\n compile group: 'org.apache.commons', name:'commons-lang3', version:'3.4'\n compile group: 'org.apache.kafka', name: 'kafka-clients', version: '0.10.1.0'\n compile group: 'org.apache.kafka', name: 'kafka-streams', version: '0.10.1.0'\n compile group: 'org.rocksdb', name: 'rocksdbjni', version: '4.1.0'\n testCompile group: 'junit', name: 'junit', version:'4.11'\n}\n" }, { "path": "gradle/wrapper/gradle-wrapper.properties", "content": "#Sat Jul 16 21:02:57 EDT 2016\ndistributionBase=GRADLE_USER_HOME\ndistributionPath=wrapper/dists\nzipStoreBase=GRADLE_USER_HOME\nzipStorePath=wrapper/dists\ndistributionUrl=https\\://services.gradle.org/distributions/gradle-2.11-all.zip\n" }, { "path": "gradlew", "content": "#!/usr/bin/env bash\n\n##############################################################################\n##\n## Gradle start up script for UN*X\n##\n##############################################################################\n\n# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.\nDEFAULT_JVM_OPTS=\"\"\n\nAPP_NAME=\"Gradle\"\nAPP_BASE_NAME=`basename \"$0\"`\n\n# Use the maximum available, or set MAX_FD != -1 to use that value.\nMAX_FD=\"maximum\"\n\nwarn ( ) {\n echo \"$*\"\n}\n\ndie ( ) {\n echo\n echo \"$*\"\n echo\n exit 1\n}\n\n# OS specific support (must be 'true' or 'false').\ncygwin=false\nmsys=false\ndarwin=false\ncase \"`uname`\" in\n CYGWIN* )\n cygwin=true\n ;;\n Darwin* )\n darwin=true\n ;;\n MINGW* )\n msys=true\n ;;\nesac\n\n# Attempt to set APP_HOME\n# Resolve links: $0 may be a link\nPRG=\"$0\"\n# Need this for relative symlinks.\nwhile [ -h \"$PRG\" ] ; do\n ls=`ls -ld \"$PRG\"`\n link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n if expr \"$link\" : '/.*' > /dev/null; then\n PRG=\"$link\"\n else\n PRG=`dirname \"$PRG\"`\"/$link\"\n fi\ndone\nSAVED=\"`pwd`\"\ncd \"`dirname \\\"$PRG\\\"`/\" >/dev/null\nAPP_HOME=\"`pwd -P`\"\ncd \"$SAVED\" >/dev/null\n\nCLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar\n\n# Determine the Java command to use to start the JVM.\nif [ -n \"$JAVA_HOME\" ] ; then\n if [ -x \"$JAVA_HOME/jre/sh/java\" ] ; then\n # IBM's JDK on AIX uses strange locations for the executables\n JAVACMD=\"$JAVA_HOME/jre/sh/java\"\n else\n JAVACMD=\"$JAVA_HOME/bin/java\"\n fi\n if [ ! -x \"$JAVACMD\" ] ; then\n die \"ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME\n\nPlease set the JAVA_HOME variable in your environment to match the\nlocation of your Java installation.\"\n fi\nelse\n JAVACMD=\"java\"\n which java >/dev/null 2>&1 || die \"ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.\n\nPlease set the JAVA_HOME variable in your environment to match the\nlocation of your Java installation.\"\nfi\n\n# Increase the maximum file descriptors if we can.\nif [ \"$cygwin\" = \"false\" -a \"$darwin\" = \"false\" ] ; then\n MAX_FD_LIMIT=`ulimit -H -n`\n if [ $? -eq 0 ] ; then\n if [ \"$MAX_FD\" = \"maximum\" -o \"$MAX_FD\" = \"max\" ] ; then\n MAX_FD=\"$MAX_FD_LIMIT\"\n fi\n ulimit -n $MAX_FD\n if [ $? -ne 0 ] ; then\n warn \"Could not set maximum file descriptor limit: $MAX_FD\"\n fi\n else\n warn \"Could not query maximum file descriptor limit: $MAX_FD_LIMIT\"\n fi\nfi\n\n# For Darwin, add options to specify how the application appears in the dock\nif $darwin; then\n GRADLE_OPTS=\"$GRADLE_OPTS \\\"-Xdock:name=$APP_NAME\\\" \\\"-Xdock:icon=$APP_HOME/media/gradle.icns\\\"\"\nfi\n\n# For Cygwin, switch paths to Windows format before running java\nif $cygwin ; then\n APP_HOME=`cygpath --path --mixed \"$APP_HOME\"`\n CLASSPATH=`cygpath --path --mixed \"$CLASSPATH\"`\n JAVACMD=`cygpath --unix \"$JAVACMD\"`\n\n # We build the pattern for arguments to be converted via cygpath\n ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`\n SEP=\"\"\n for dir in $ROOTDIRSRAW ; do\n ROOTDIRS=\"$ROOTDIRS$SEP$dir\"\n SEP=\"|\"\n done\n OURCYGPATTERN=\"(^($ROOTDIRS))\"\n # Add a user-defined pattern to the cygpath arguments\n if [ \"$GRADLE_CYGPATTERN\" != \"\" ] ; then\n OURCYGPATTERN=\"$OURCYGPATTERN|($GRADLE_CYGPATTERN)\"\n fi\n # Now convert the arguments - kludge to limit ourselves to /bin/sh\n i=0\n for arg in \"$@\" ; do\n CHECK=`echo \"$arg\"|egrep -c \"$OURCYGPATTERN\" -`\n CHECK2=`echo \"$arg\"|egrep -c \"^-\"` ### Determine if an option\n\n if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition\n eval `echo args$i`=`cygpath --path --ignore --mixed \"$arg\"`\n else\n eval `echo args$i`=\"\\\"$arg\\\"\"\n fi\n i=$((i+1))\n done\n case $i in\n (0) set -- ;;\n (1) set -- \"$args0\" ;;\n (2) set -- \"$args0\" \"$args1\" ;;\n (3) set -- \"$args0\" \"$args1\" \"$args2\" ;;\n (4) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" ;;\n (5) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" ;;\n (6) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" ;;\n (7) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" ;;\n (8) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" \"$args7\" ;;\n (9) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" \"$args7\" \"$args8\" ;;\n esac\nfi\n\n# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules\nfunction splitJvmOpts() {\n JVM_OPTS=(\"$@\")\n}\neval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS\nJVM_OPTS[${#JVM_OPTS[*]}]=\"-Dorg.gradle.appname=$APP_BASE_NAME\"\n\nexec \"$JAVACMD\" \"${JVM_OPTS[@]}\" -classpath \"$CLASSPATH\" org.gradle.wrapper.GradleWrapperMain \"$@\"\n" }, { "path": "gradlew.bat", "content": "@if \"%DEBUG%\" == \"\" @echo off\n@rem ##########################################################################\n@rem\n@rem Gradle startup script for Windows\n@rem\n@rem ##########################################################################\n\n@rem Set local scope for the variables with windows NT shell\nif \"%OS%\"==\"Windows_NT\" setlocal\n\n@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.\nset DEFAULT_JVM_OPTS=\n\nset DIRNAME=%~dp0\nif \"%DIRNAME%\" == \"\" set DIRNAME=.\nset APP_BASE_NAME=%~n0\nset APP_HOME=%DIRNAME%\n\n@rem Find java.exe\nif defined JAVA_HOME goto findJavaFromJavaHome\n\nset JAVA_EXE=java.exe\n%JAVA_EXE% -version >NUL 2>&1\nif \"%ERRORLEVEL%\" == \"0\" goto init\n\necho.\necho ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.\necho.\necho Please set the JAVA_HOME variable in your environment to match the\necho location of your Java installation.\n\ngoto fail\n\n:findJavaFromJavaHome\nset JAVA_HOME=%JAVA_HOME:\"=%\nset JAVA_EXE=%JAVA_HOME%/bin/java.exe\n\nif exist \"%JAVA_EXE%\" goto init\n\necho.\necho ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%\necho.\necho Please set the JAVA_HOME variable in your environment to match the\necho location of your Java installation.\n\ngoto fail\n\n:init\n@rem Get command-line arguments, handling Windows variants\n\nif not \"%OS%\" == \"Windows_NT\" goto win9xME_args\nif \"%@eval[2+2]\" == \"4\" goto 4NT_args\n\n:win9xME_args\n@rem Slurp the command line arguments.\nset CMD_LINE_ARGS=\nset _SKIP=2\n\n:win9xME_args_slurp\nif \"x%~1\" == \"x\" goto execute\n\nset CMD_LINE_ARGS=%*\ngoto execute\n\n:4NT_args\n@rem Get arguments from the 4NT Shell from JP Software\nset CMD_LINE_ARGS=%$\n\n:execute\n@rem Setup the command line\n\nset CLASSPATH=%APP_HOME%\\gradle\\wrapper\\gradle-wrapper.jar\n\n@rem Execute Gradle\n\"%JAVA_EXE%\" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% \"-Dorg.gradle.appname=%APP_BASE_NAME%\" -classpath \"%CLASSPATH%\" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%\n\n:end\n@rem End local scope for the variables with windows NT shell\nif \"%ERRORLEVEL%\"==\"0\" goto mainEnd\n\n:fail\nrem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of\nrem the _cmd.exe /c_ return code!\nif not \"\" == \"%GRADLE_EXIT_CONSOLE%\" exit 1\nexit /b 1\n\n:mainEnd\nif \"%OS%\"==\"Windows_NT\" endlocal\n\n:omega\n" }, { "path": "settings.gradle", "content": "rootProject.name = 'kafka-streams'\n" }, { "path": "src/main/java/bbejeck/model/Purchase.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.model;\n\nimport java.util.Date;\nimport java.util.Objects;\n\n/**\n * User: Bill Bejeck\n * Date: 2/20/16\n * Time: 9:09 AM\n */\npublic class Purchase {\n\n private String firstName;\n private String lastName;\n private String creditCardNumber;\n private String itemPurchased;\n int quantity;\n double price;\n private Date purchaseDate;\n private String zipCode;\n\n private Purchase(Builder builder) {\n firstName = builder.firstName;\n lastName = builder.lastName;\n creditCardNumber = builder.creditCardNumber;\n itemPurchased = builder.itemPurchased;\n quantity = builder.quanity;\n price = builder.price;\n purchaseDate = builder.purchaseDate;\n zipCode = builder.zipCode;\n }\n\n public static Builder builder() {\n return new Builder();\n }\n\n public static Builder builder(Purchase copy) {\n Builder builder = new Builder();\n builder.firstName = copy.firstName;\n builder.lastName = copy.lastName;\n builder.creditCardNumber = copy.creditCardNumber;\n builder.itemPurchased = copy.itemPurchased;\n builder.quanity = copy.quantity;\n builder.price = copy.price;\n builder.purchaseDate = copy.purchaseDate;\n builder.zipCode = copy.zipCode;\n return builder;\n }\n\n\n public String getFirstName() {\n return firstName;\n }\n\n public String getLastName() {\n return lastName;\n }\n\n public String getCreditCardNumber() {\n return creditCardNumber;\n }\n\n public String getItemPurchased() {\n return itemPurchased;\n }\n\n public int getQuantity() {\n return quantity;\n }\n\n public double getPrice() {\n return price;\n }\n\n public Date getPurchaseDate() {\n return purchaseDate;\n }\n\n public String getZipCode() {\n return zipCode;\n }\n\n @Override\n public String toString() {\n return \"Purchase{\" +\n \"firstName='\" + firstName + '\\'' +\n \", lastName='\" + lastName + '\\'' +\n \", creditCardNumber='\" + creditCardNumber + '\\'' +\n \", itemPurchased='\" + itemPurchased + '\\'' +\n \", quantity=\" + quantity +\n \", price=\" + price +\n \", purchaseDate=\" + purchaseDate +\n \", zipCode='\" + zipCode + '\\'' +\n '}';\n }\n\n public static final class Builder {\n private String firstName;\n private String lastName;\n private String creditCardNumber;\n private String itemPurchased;\n private int quanity;\n private double price;\n private Date purchaseDate;\n private String zipCode;\n\n private static final String CC_NUMBER_REPLACEMENT=\"xxxx-xxxx-xxxx-\";\n\n private Builder() {\n }\n\n public Builder firstName(String val) {\n firstName = val;\n return this;\n }\n\n public Builder lastName(String val) {\n lastName = val;\n return this;\n }\n\n\n public Builder maskCreditCard(){\n Objects.requireNonNull(this.creditCardNumber, \"Credit Card can't be null\");\n String last4Digits = this.creditCardNumber.split(\"-\")[3];\n this.creditCardNumber = CC_NUMBER_REPLACEMENT+last4Digits;\n return this;\n }\n\n public Builder creditCardNumber(String val) {\n creditCardNumber = val;\n return this;\n }\n\n public Builder itemPurchased(String val) {\n itemPurchased = val;\n return this;\n }\n\n public Builder quanity(int val) {\n quanity = val;\n return this;\n }\n\n public Builder price(double val) {\n price = val;\n return this;\n }\n\n public Builder purchaseDate(Date val) {\n purchaseDate = val;\n return this;\n }\n\n public Builder zipCode(String val) {\n zipCode = val;\n return this;\n }\n\n public Purchase build() {\n return new Purchase(this);\n }\n }\n}\n" }, { "path": "src/main/java/bbejeck/model/PurchasePattern.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.model;\n\nimport java.util.Date;\n\n/**\n * User: Bill Bejeck\n * Date: 2/21/16\n * Time: 3:36 PM\n */\npublic class PurchasePattern {\n\n private String zipCode;\n private String item;\n private Date date;\n\n private PurchasePattern(Builder builder) {\n zipCode = builder.zipCode;\n item = builder.item;\n date = builder.date;\n }\n\n public static Builder newBuilder() {\n return new Builder();\n }\n\n public static Builder builder(Purchase purchase){\n return new Builder(purchase);\n\n }\n public String getZipCode() {\n return zipCode;\n }\n\n public String getItem() {\n return item;\n }\n\n public Date getDate() {\n return date;\n }\n\n\n @Override\n public String toString() {\n return \"PurchasePattern{\" +\n \"zipCode='\" + zipCode + '\\'' +\n \", item='\" + item + '\\'' +\n \", date=\" + date +\n '}';\n }\n\n public static final class Builder {\n private String zipCode;\n private String item;\n private Date date;\n\n private Builder() {\n }\n\n private Builder(Purchase purchase) {\n this.zipCode = purchase.getZipCode();\n this.item = purchase.getItemPurchased();\n this.date = purchase.getPurchaseDate();\n }\n\n public Builder zipCode(String val) {\n zipCode = val;\n return this;\n }\n\n public Builder item(String val) {\n item = val;\n return this;\n }\n\n public Builder date(Date val) {\n date = val;\n return this;\n }\n\n public PurchasePattern build() {\n return new PurchasePattern(this);\n }\n }\n}\n" }, { "path": "src/main/java/bbejeck/model/RewardAccumulator.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.model;\n\n/**\n * User: Bill Bejeck\n * Date: 2/20/16\n * Time: 9:55 AM\n */\npublic class RewardAccumulator {\n\n private String customerName;\n private double purchaseTotal;\n\n private RewardAccumulator(String customerName, double purchaseTotal) {\n this.customerName = customerName;\n this.purchaseTotal = purchaseTotal;\n }\n\n public String getCustomerName() {\n return customerName;\n }\n\n public double getPurchaseTotal() {\n return purchaseTotal;\n }\n\n @Override\n public String toString() {\n return \"RewardAccumulator{\" +\n \"customerName='\" + customerName + '\\'' +\n \", purchaseTotal=\" + purchaseTotal +\n '}';\n }\n\n public static Builder builder(Purchase purchase){return new Builder(purchase);}\n\n public static final class Builder {\n private String customerName;\n private double purchaseTotal;\n\n private Builder(Purchase purchase){\n this.customerName = purchase.getLastName()+\",\"+purchase.getFirstName();\n this.purchaseTotal = purchase.getPrice() * purchase.getQuantity();\n }\n\n\n public RewardAccumulator build(){\n return new RewardAccumulator(customerName,purchaseTotal);\n }\n\n }\n}\n" }, { "path": "src/main/java/bbejeck/model/StockTransaction.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.model;\n\nimport java.util.Date;\n\n/**\n * User: Bill Bejeck\n * Date: 2/14/16\n * Time: 2:31 PM\n */\npublic class StockTransaction {\n\n private String symbol;\n private String type;\n private double shares;\n private double amount;\n private Date timeStamp;\n\n\n public String getSymbol() {\n return symbol;\n }\n\n public void setSymbol(String symbol) {\n this.symbol = symbol;\n }\n\n public String getType() {\n return type;\n }\n\n public void setType(String type) {\n this.type = type;\n }\n\n public double getShares() {\n return shares;\n }\n\n public void setShares(double shares) {\n this.shares = shares;\n }\n\n public double getAmount() {\n return amount;\n }\n\n public void setAmount(double amount) {\n this.amount = amount;\n }\n\n public Date getTimeStamp() {\n return timeStamp;\n }\n\n public void setTimeStamp(Date timeStamp) {\n this.timeStamp = timeStamp;\n }\n\n @Override\n public String toString() {\n return \"StockTransaction{\" +\n \"symbol='\" + symbol + '\\'' +\n \", type='\" + type + '\\'' +\n \", shares=\" + shares +\n \", amount=\" + amount +\n \", timeStamp=\" + timeStamp +\n '}';\n }\n}\n" }, { "path": "src/main/java/bbejeck/model/StockTransactionCollector.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.model;\n\n/**\n * User: Bill Bejeck\n * Date: 3/10/16\n * Time: 7:54 PM\n */\npublic class StockTransactionCollector {\n\n private double amount;\n private String tickerSymbol;\n private int sharesPurchased;\n private int sharesSold;\n\n public StockTransactionCollector add(StockTransaction transaction){\n if(tickerSymbol == null){\n tickerSymbol = transaction.getSymbol();\n }\n\n this.amount += transaction.getAmount();\n if(transaction.getType().equalsIgnoreCase(\"purchase\")){\n this.sharesPurchased += transaction.getShares();\n } else{\n this.sharesSold += transaction.getShares();\n }\n return this;\n }\n\n @Override\n public String toString() {\n return \"StockTransactionCollector{\" +\n \"amount=\" + amount +\n \", tickerSymbol='\" + tickerSymbol + '\\'' +\n \", sharesPurchased=\" + sharesPurchased +\n \", sharesSold=\" + sharesSold +\n '}';\n }\n}\n" }, { "path": "src/main/java/bbejeck/model/StockTransactionSummary.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.model;\n\n/**\n * User: Bill Bejeck\n * Date: 2/6/16\n * Time: 3:32 PM\n */\npublic class StockTransactionSummary {\n\n public double amount;\n public String tickerSymbol;\n public int sharesPurchased;\n public int sharesSold;\n private long lastUpdatedTime;\n\n\n\n public void update(StockTransaction transaction){\n this.amount += transaction.getAmount();\n if(transaction.getType().equalsIgnoreCase(\"purchase\")){\n this.sharesPurchased += transaction.getShares();\n } else{\n this.sharesSold += transaction.getShares();\n }\n this.lastUpdatedTime = System.currentTimeMillis();\n }\n\n public boolean updatedWithinLastMillis(long currentTime, long limit){\n return currentTime - this.lastUpdatedTime <= limit;\n }\n\n public static StockTransactionSummary fromTransaction(StockTransaction transaction){\n StockTransactionSummary summary = new StockTransactionSummary();\n summary.tickerSymbol = transaction.getSymbol();\n summary.update(transaction);\n return summary;\n }\n}\n" }, { "path": "src/main/java/bbejeck/model/Tweet.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.model;\n\n/**\n * User: Bill Bejeck\n * Date: 4/21/16\n * Time: 8:38 PM\n */\npublic class Tweet {\n\n private String id;\n private String text;\n private String language;\n\n public String getId() {\n return id;\n }\n\n public void setId(String id) {\n this.id = id;\n }\n\n public String getText() {\n return text;\n }\n\n public void setText(String text) {\n this.text = text;\n }\n\n public String getLanguage() {\n return language;\n }\n\n public void setLanguage(String language) {\n this.language = language;\n }\n\n\n @Override\n public String toString() {\n return \"Tweet{\" +\n \"id='\" + id + '\\'' +\n \", text='\" + text + '\\'' +\n \", language='\" + language + '\\'' +\n '}';\n }\n}\n" }, { "path": "src/main/java/bbejeck/nlp/Classifier.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.nlp;\n\nimport com.aliasi.classify.Classification;\nimport com.aliasi.classify.Classified;\nimport com.aliasi.classify.DynamicLMClassifier;\nimport com.aliasi.lm.NGramBoundaryLM;\n\nimport java.io.BufferedReader;\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.IOException;\nimport java.io.InputStreamReader;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\n\n/**\n * User: Bill Bejeck\n * Date: 4/20/16\n * Time: 10:01 PM\n */\npublic class Classifier {\n\n private DynamicLMClassifier classifier;\n private int maxCharNGram = 3;\n private String trainingDataDelimiter;\n\n public Classifier(String trainingDataDelimiter) {\n this.trainingDataDelimiter = trainingDataDelimiter;\n }\n\n public Classifier(){\n this(\"#\");\n }\n\n public void train(File trainingData) {\n Set categorySet = new HashSet<>();\n List annotatedData = new ArrayList<>();\n fillCategoriesAndAnnotatedData(trainingData, categorySet, annotatedData);\n trainClassifier(categorySet, annotatedData);\n }\n\n private void trainClassifier(Set categorySet, List annotatedData){\n String[] categories = categorySet.toArray(new String[0]);\n classifier = DynamicLMClassifier.createNGramBoundary(categories,maxCharNGram);\n for (String[] row: annotatedData) {\n String actualClassification = row[0];\n String text = row[1];\n Classification classification = new Classification(actualClassification);\n Classified classified = new Classified<>(text,classification);\n classifier.handle(classified);\n }\n }\n\n\n private void fillCategoriesAndAnnotatedData(File trainingData,\n Set categorySet,\n List annotatedData) {\n try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(trainingData)))) {\n\n String line = reader.readLine();\n while (line != null) {\n String[] data = line.split(trainingDataDelimiter);\n categorySet.add(data[0]);\n annotatedData.add(data);\n line = reader.readLine();\n }\n\n } catch (IOException e){\n throw new RuntimeException(e);\n }\n }\n\n\n public String classify(String text){\n return classifier.classify(text.trim()).bestCategory().toLowerCase();\n }\n\n}\n" }, { "path": "src/main/java/bbejeck/nlp/LingPipeTester.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.nlp;\n\nimport com.aliasi.classify.Classification;\nimport com.aliasi.classify.Classified;\nimport com.aliasi.classify.DynamicLMClassifier;\nimport com.aliasi.lm.NGramBoundaryLM;\n\nimport java.io.BufferedReader;\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.InputStreamReader;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\n\n/**\n * User: Bill Bejeck\n * Date: 4/9/16\n * Time: 6:24 PM\n */\npublic class LingPipeTester {\n\n public static void main(String[] args) throws Exception {\n File trainingData = new File(\"src/main/resources/kafkaStreamsTwitterTrainingData_clean.csv\");\n int maxCharNGram = 3;\n\n BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(trainingData)));\n Set categorySet = new HashSet<>();\n List annotatedData = new ArrayList<>();\n String line = reader.readLine();\n while (line !=null){\n String[] data = line.split(\"#\");\n categorySet.add(data[0]);\n annotatedData.add(data);\n line = reader.readLine();\n }\n System.out.println(\"read in all data\");\n reader.close();\n String[] categories = categorySet.toArray(new String[0]);\n\n DynamicLMClassifier classifier\n = DynamicLMClassifier.createNGramBoundary(categories,maxCharNGram);\n for (String[] row: annotatedData) {\n String truth = row[0];\n String text = row[1];\n Classification classification = new Classification(truth);\n Classified classified = new Classified<>(text,classification);\n classifier.handle(classified);\n }\n System.out.println(\"training complete\");\n\n reader = new BufferedReader(new InputStreamReader(System.in));\n\n System.out.println(\"enter text, I'll tell you the language\");\n String text;\n while (!(text = reader.readLine()).equalsIgnoreCase(\"quit\")) {\n Classification classification = classifier.classify(text);\n System.out.println(\"Entered -> \" + text);\n System.out.println(\"lang -> \" + classification.bestCategory());\n }\n }\n}\n" }, { "path": "src/main/java/bbejeck/processor/purchases/CreditCardAnonymizer.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.processor.purchases;\n\nimport bbejeck.model.Purchase;\nimport org.apache.kafka.streams.processor.AbstractProcessor;\n\n/**\n * User: Bill Bejeck\n * Date: 2/20/16\n * Time: 9:19 AM\n */\npublic class CreditCardAnonymizer extends AbstractProcessor {\n\n private static final String CC_NUMBER_REPLACEMENT=\"xxxx-xxxx-xxxx-\";\n\n @Override\n public void process(String key, Purchase purchase) {\n String last4Digits = purchase.getCreditCardNumber().split(\"-\")[3];\n Purchase updated = Purchase.builder(purchase).creditCardNumber(CC_NUMBER_REPLACEMENT+last4Digits).build();\n context().forward(key,updated);\n context().commit();\n }\n}\n" }, { "path": "src/main/java/bbejeck/processor/purchases/CustomerRewards.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.processor.purchases;\n\nimport bbejeck.model.Purchase;\nimport bbejeck.model.RewardAccumulator;\nimport org.apache.kafka.streams.processor.AbstractProcessor;\n\n/**\n * User: Bill Bejeck\n * Date: 2/20/16\n * Time: 9:44 AM\n */\npublic class CustomerRewards extends AbstractProcessor {\n\n @Override\n public void process(String key, Purchase value) {\n RewardAccumulator accumulator = RewardAccumulator.builder(value).build();\n context().forward(key,accumulator);\n context().commit();\n\n }\n}\n" }, { "path": "src/main/java/bbejeck/processor/purchases/PurchasePatterns.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.processor.purchases;\n\nimport bbejeck.model.Purchase;\nimport bbejeck.model.PurchasePattern;\nimport org.apache.kafka.streams.processor.AbstractProcessor;\n\n/**\n * User: Bill Bejeck\n * Date: 2/20/16\n * Time: 9:39 AM\n */\npublic class PurchasePatterns extends AbstractProcessor {\n\n @Override\n public void process(String key, Purchase value) {\n PurchasePattern purchasePattern = PurchasePattern.newBuilder().date(value.getPurchaseDate())\n .item(value.getItemPurchased())\n .zipCode(value.getZipCode()).build();\n context().forward(key, purchasePattern);\n context().commit();\n }\n}\n" }, { "path": "src/main/java/bbejeck/processor/purchases/PurchaseProcessorDriver.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.processor.purchases;\n\nimport bbejeck.model.Purchase;\nimport bbejeck.model.PurchasePattern;\nimport bbejeck.model.RewardAccumulator;\nimport bbejeck.serializer.JsonDeserializer;\nimport bbejeck.serializer.JsonSerializer;\nimport org.apache.kafka.common.serialization.StringDeserializer;\nimport org.apache.kafka.common.serialization.StringSerializer;\nimport org.apache.kafka.streams.KafkaStreams;\nimport org.apache.kafka.streams.StreamsConfig;\nimport org.apache.kafka.streams.processor.TopologyBuilder;\nimport org.apache.kafka.streams.processor.WallclockTimestampExtractor;\n\nimport java.util.Properties;\n\n/**\n * User: Bill Bejeck\n * Date: 11/5/15\n * Time: 10:22 PM\n */\npublic class PurchaseProcessorDriver {\n\n public static void main(String[] args) throws Exception {\n\n StreamsConfig streamingConfig = new StreamsConfig(getProperties());\n\n JsonDeserializer purchaseJsonDeserializer = new JsonDeserializer<>(Purchase.class);\n JsonSerializer purchaseJsonSerializer = new JsonSerializer<>();\n JsonSerializer rewardAccumulatorJsonSerializer = new JsonSerializer<>();\n JsonSerializer purchasePatternJsonSerializer = new JsonSerializer<>();\n\n StringDeserializer stringDeserializer = new StringDeserializer();\n StringSerializer stringSerializer = new StringSerializer();\n\n TopologyBuilder topologyBuilder = new TopologyBuilder();\n topologyBuilder.addSource(\"SOURCE\", stringDeserializer, purchaseJsonDeserializer, \"src-topic\")\n\n .addProcessor(\"PROCESS\", CreditCardAnonymizer::new, \"SOURCE\")\n .addProcessor(\"PROCESS2\", PurchasePatterns::new, \"PROCESS\")\n .addProcessor(\"PROCESS3\", CustomerRewards::new, \"PROCESS\")\n\n .addSink(\"SINK\", \"patterns\", stringSerializer, purchasePatternJsonSerializer, \"PROCESS2\")\n .addSink(\"SINK2\", \"rewards\",stringSerializer, rewardAccumulatorJsonSerializer, \"PROCESS3\")\n .addSink(\"SINK3\", \"purchases\", stringSerializer, purchaseJsonSerializer, \"PROCESS\");\n\n System.out.println(\"Starting PurchaseProcessor Example\");\n KafkaStreams streaming = new KafkaStreams(topologyBuilder, streamingConfig);\n streaming.start();\n System.out.println(\"Now started PurchaseProcessor Example\");\n\n }\n\n private static Properties getProperties() {\n Properties props = new Properties();\n props.put(StreamsConfig.CLIENT_ID_CONFIG, \"Example-Processor-Job\");\n props.put(\"group.id\", \"test-consumer-group\");\n props.put(StreamsConfig.APPLICATION_ID_CONFIG, \"testing-processor-api\");\n props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, \"localhost:9092\");\n props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, \"localhost:2181\");\n props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 1);\n props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class);\n return props;\n }\n}" }, { "path": "src/main/java/bbejeck/processor/stocks/StockSummaryProcessor.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.processor.stocks;\n\nimport bbejeck.model.StockTransaction;\nimport bbejeck.model.StockTransactionSummary;\nimport org.apache.kafka.streams.processor.AbstractProcessor;\nimport org.apache.kafka.streams.processor.ProcessorContext;\nimport org.apache.kafka.streams.state.KeyValueIterator;\nimport org.apache.kafka.streams.state.KeyValueStore;\n\nimport java.util.Objects;\n\n/**\n * User: Bill Bejeck\n * Date: 1/25/16\n * Time: 8:13 PM\n */\n\n@SuppressWarnings(\"unchecked\")\npublic class StockSummaryProcessor extends AbstractProcessor {\n\n private KeyValueStore summaryStore;\n private ProcessorContext context;\n\n\n public void process(String key, StockTransaction stockTransaction) {\n String currentSymbol = stockTransaction.getSymbol();\n StockTransactionSummary transactionSummary = summaryStore.get(currentSymbol);\n if (transactionSummary == null) {\n transactionSummary = StockTransactionSummary.fromTransaction(stockTransaction);\n } else {\n transactionSummary.update(stockTransaction);\n }\n summaryStore.put(currentSymbol, transactionSummary);\n\n this.context.commit();\n }\n\n\n @Override\n @SuppressWarnings(\"unchecked\")\n public void init(ProcessorContext context) {\n this.context = context;\n this.context.schedule(10000);\n summaryStore = (KeyValueStore) this.context.getStateStore(\"stock-transactions\");\n Objects.requireNonNull(summaryStore, \"State store can't be null\");\n\n }\n\n\n @Override\n public void punctuate(long streamTime) {\n KeyValueIterator it = summaryStore.all();\n long currentTime = System.currentTimeMillis();\n while (it.hasNext()) {\n StockTransactionSummary summary = it.next().value;\n if (summary.updatedWithinLastMillis(currentTime, 11000)) {\n this.context.forward(summary.tickerSymbol, summary);\n }\n }\n }\n\n}\n" }, { "path": "src/main/java/bbejeck/processor/stocks/StockSummaryStatefulProcessorDriver.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.processor.stocks;\n\nimport bbejeck.model.StockTransaction;\nimport bbejeck.model.StockTransactionSummary;\nimport bbejeck.serializer.JsonDeserializer;\nimport bbejeck.serializer.JsonSerializer;\nimport org.apache.kafka.common.serialization.Serde;\nimport org.apache.kafka.common.serialization.Serdes;\nimport org.apache.kafka.common.serialization.StringDeserializer;\nimport org.apache.kafka.common.serialization.StringSerializer;\nimport org.apache.kafka.streams.KafkaStreams;\nimport org.apache.kafka.streams.StreamsConfig;\nimport org.apache.kafka.streams.processor.TopologyBuilder;\nimport org.apache.kafka.streams.processor.WallclockTimestampExtractor;\nimport org.apache.kafka.streams.state.Stores;\n\nimport java.util.Properties;\n\n/**\n * User: Bill Bejeck\n * Date: 2/8/16\n * Time: 5:11 PM\n */\npublic class StockSummaryStatefulProcessorDriver {\n\n public static void main(String[] args) {\n\n StreamsConfig streamingConfig = new StreamsConfig(getProperties());\n\n TopologyBuilder builder = new TopologyBuilder();\n\n JsonSerializer stockTxnSummarySerializer = new JsonSerializer<>();\n JsonDeserializer stockTxnSummaryDeserializer = new JsonDeserializer<>(StockTransactionSummary.class);\n JsonDeserializer stockTxnDeserializer = new JsonDeserializer<>(StockTransaction.class);\n JsonSerializer stockTxnJsonSerializer = new JsonSerializer<>();\n StringSerializer stringSerializer = new StringSerializer();\n StringDeserializer stringDeserializer = new StringDeserializer();\n\n Serde stockTransactionSummarySerde = Serdes.serdeFrom(stockTxnSummarySerializer,stockTxnSummaryDeserializer);\n\n builder.addSource(\"stocks-source\", stringDeserializer, stockTxnDeserializer, \"stocks\")\n .addProcessor(\"summary\", StockSummaryProcessor::new, \"stocks-source\")\n .addStateStore(Stores.create(\"stock-transactions\").withStringKeys()\n .withValues(stockTransactionSummarySerde).inMemory().maxEntries(100).build(),\"summary\")\n .addSink(\"sink\", \"stocks-out\", stringSerializer,stockTxnJsonSerializer,\"stocks-source\")\n .addSink(\"sink-2\", \"transaction-summary\", stringSerializer, stockTxnSummarySerializer, \"summary\");\n\n System.out.println(\"Starting StockSummaryStatefulProcessor Example\");\n KafkaStreams streaming = new KafkaStreams(builder, streamingConfig);\n streaming.start();\n System.out.println(\"StockSummaryStatefulProcessor Example now started\");\n\n }\n\n private static Properties getProperties() {\n Properties props = new Properties();\n props.put(StreamsConfig.CLIENT_ID_CONFIG, \"Sample-Stateful-Processor\");\n props.put(\"group.id\", \"test-consumer-group\");\n props.put(StreamsConfig.APPLICATION_ID_CONFIG, \"stateful_processor_id\");\n props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, \"localhost:9092\");\n props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, \"localhost:2181\");\n props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 1);\n props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class);\n return props;\n }\n}\n" }, { "path": "src/main/java/bbejeck/processor/twitter/TwitterClassificationProcessor.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.processor.twitter;\n\nimport bbejeck.model.Tweet;\nimport org.apache.kafka.streams.processor.AbstractProcessor;\n\n/**\n * User: Bill Bejeck\n * Date: 4/21/16\n * Time: 9:10 PM\n */\npublic class TwitterClassificationProcessor extends AbstractProcessor {\n\n\n\n @Override\n public void process(String s, Tweet tweet) {\n\n }\n}\n" }, { "path": "src/main/java/bbejeck/serializer/JsonDeserializer.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.serializer;\n\nimport com.google.gson.Gson;\nimport org.apache.kafka.common.serialization.Deserializer;\n\nimport java.util.Map;\n\n/**\n * User: Bill Bejeck\n * Date: 2/14/16\n * Time: 3:26 PM\n */\n\npublic class JsonDeserializer implements Deserializer {\n\n private Gson gson = new Gson();\n private Class deserializedClass;\n\n public JsonDeserializer(Class deserializedClass) {\n this.deserializedClass = deserializedClass;\n }\n\n public JsonDeserializer() {\n }\n\n @Override\n @SuppressWarnings(\"unchecked\")\n public void configure(Map map, boolean b) {\n if(deserializedClass == null) {\n deserializedClass = (Class) map.get(\"serializedClass\");\n }\n }\n\n @Override\n public T deserialize(String s, byte[] bytes) {\n if(bytes == null){\n return null;\n }\n\n return gson.fromJson(new String(bytes),deserializedClass);\n\n }\n\n @Override\n public void close() {\n\n }\n}\n" }, { "path": "src/main/java/bbejeck/serializer/JsonSerializer.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.serializer;\n\nimport com.google.gson.Gson;\nimport org.apache.kafka.common.serialization.Serializer;\n\nimport java.nio.charset.Charset;\nimport java.util.Map;\n\n/**\n * User: Bill Bejeck\n * Date: 2/14/16\n * Time: 2:37 PM\n */\npublic class JsonSerializer implements Serializer {\n\n private Gson gson = new Gson();\n\n @Override\n public void configure(Map map, boolean b) {\n\n }\n\n @Override\n public byte[] serialize(String topic, T t) {\n return gson.toJson(t).getBytes(Charset.forName(\"UTF-8\"));\n }\n\n @Override\n public void close() {\n\n }\n}\n" }, { "path": "src/main/java/bbejeck/streams/RegexTest.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.streams;\n\nimport org.apache.kafka.clients.consumer.ConsumerConfig;\nimport org.apache.kafka.common.serialization.Serdes;\nimport org.apache.kafka.streams.KafkaStreams;\nimport org.apache.kafka.streams.StreamsConfig;\nimport org.apache.kafka.streams.kstream.KStream;\nimport org.apache.kafka.streams.kstream.KStreamBuilder;\nimport org.apache.kafka.streams.processor.WallclockTimestampExtractor;\n\nimport java.util.Properties;\nimport java.util.regex.Pattern;\n\n/**\n * Example on Using Regex Patterns For Source Topics\n */\npublic class RegexTest {\n\n\n public static void main(String[] args) {\n\n StreamsConfig streamingConfig = new StreamsConfig(getProperties());\n KStreamBuilder kStreamBuilder = new KStreamBuilder();\n\n\n KStream patternStreamI = kStreamBuilder.stream(Serdes.String(), Serdes.String(), Pattern.compile(\"topic-\\\\d\"));\n KStream namedTopicKStream = kStreamBuilder.stream(Serdes.String(), Serdes.String(), \"topic-Z\");\n KStream patternStreamII = kStreamBuilder.stream(Serdes.String(), Serdes.String(), Pattern.compile(\"topic-[A-Y]+\"));\n\n patternStreamI.print(\"pattern-\\\\d\");\n namedTopicKStream.print(\"topic-Z\");\n patternStreamII.print(\"topic-[A-Y]+\");\n\n\n System.out.println(\"Starting stream regex consumer Example\");\n KafkaStreams kafkaStreams = new KafkaStreams(kStreamBuilder, streamingConfig);\n kafkaStreams.start();\n\n\n }\n\n\n private static Properties getProperties() {\n Properties props = new Properties();\n props.put(StreamsConfig.APPLICATION_ID_CONFIG, \"regex_id\");\n props.put(ConsumerConfig.GROUP_ID_CONFIG, \"regex_group_id\");\n props.put(ConsumerConfig.CLIENT_ID_CONFIG, \"regex_id\");\n props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, \"earliest\");\n props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, \"5000\");\n props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, \"localhost:9092\");\n props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, \"localhost:2181\");\n props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, \"2\");\n props.put(ConsumerConfig.METADATA_MAX_AGE_CONFIG, \"15000\");\n props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, \"org.apache.kafka.common.serialization.StringDeserializer\");\n props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, \"org.apache.kafka.common.serialization.StringDeserializer\");\n props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 1);\n props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class);\n return props;\n }\n}\n" }, { "path": "src/main/java/bbejeck/streams/purchases/PurchaseKafkaStreamsDriver.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.streams.purchases;\n\nimport bbejeck.model.Purchase;\nimport bbejeck.model.PurchasePattern;\nimport bbejeck.model.RewardAccumulator;\nimport bbejeck.serializer.JsonDeserializer;\nimport bbejeck.serializer.JsonSerializer;\nimport org.apache.kafka.common.serialization.Serde;\nimport org.apache.kafka.common.serialization.Serdes;\nimport org.apache.kafka.common.serialization.StringDeserializer;\nimport org.apache.kafka.common.serialization.StringSerializer;\nimport org.apache.kafka.streams.KafkaStreams;\nimport org.apache.kafka.streams.StreamsConfig;\nimport org.apache.kafka.streams.kstream.KStream;\nimport org.apache.kafka.streams.kstream.KStreamBuilder;\nimport org.apache.kafka.streams.processor.WallclockTimestampExtractor;\n\nimport java.util.Properties;\n\n/**\n * User: Bill Bejeck\n * Date: 3/7/16\n * Time: 5:34 PM\n */\npublic class PurchaseKafkaStreamsDriver {\n\n public static void main(String[] args) {\n\n\n StreamsConfig streamsConfig = new StreamsConfig(getProperties());\n\n JsonDeserializer purchaseJsonDeserializer = new JsonDeserializer<>(Purchase.class);\n JsonSerializer purchaseJsonSerializer = new JsonSerializer<>();\n\n JsonSerializer rewardAccumulatorJsonSerializer = new JsonSerializer<>();\n JsonDeserializer rewardAccumulatorJsonDeserializer = new JsonDeserializer<>(RewardAccumulator.class);\n\n Serde rewardAccumulatorSerde = Serdes.serdeFrom(rewardAccumulatorJsonSerializer,rewardAccumulatorJsonDeserializer);\n\n JsonSerializer purchasePatternJsonSerializer = new JsonSerializer<>();\n JsonDeserializer purchasePatternJsonDeserializer = new JsonDeserializer<>(PurchasePattern.class);\n\n Serde purchasePatternSerde = Serdes.serdeFrom(purchasePatternJsonSerializer,purchasePatternJsonDeserializer);\n\n Serde purchaseSerde = Serdes.serdeFrom(purchaseJsonSerializer,purchaseJsonDeserializer);\n\n Serde stringSerde = Serdes.String();\n\n KStreamBuilder kStreamBuilder = new KStreamBuilder();\n\n\n KStream purchaseKStream = kStreamBuilder.stream(stringSerde,purchaseSerde,\"src-topic\")\n .mapValues(p -> Purchase.builder(p).maskCreditCard().build());\n\n purchaseKStream.mapValues(purchase -> PurchasePattern.builder(purchase).build()).to(stringSerde,purchasePatternSerde,\"patterns\");\n\n purchaseKStream.mapValues(purchase -> RewardAccumulator.builder(purchase).build()).to(stringSerde,rewardAccumulatorSerde,\"rewards\");\n\n purchaseKStream.to(stringSerde,purchaseSerde,\"purchases\");\n\n System.out.println(\"Starting PurchaseStreams Example\");\n KafkaStreams kafkaStreams = new KafkaStreams(kStreamBuilder,streamsConfig);\n kafkaStreams.start();\n System.out.println(\"Now started PurchaseStreams Example\");\n\n }\n\n\n\n\n private static Properties getProperties() {\n Properties props = new Properties();\n props.put(StreamsConfig.CLIENT_ID_CONFIG, \"Example-Kafka-Streams-Job\");\n props.put(\"group.id\", \"streams-purchases\");\n props.put(StreamsConfig.APPLICATION_ID_CONFIG, \"testing-streams-api\");\n props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, \"localhost:9092\");\n props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, \"localhost:2181\");\n props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 1);\n props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class);\n return props;\n }\n\n}\n" }, { "path": "src/main/java/bbejeck/streams/stocks/StocksKafkaStreamsDriver.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.streams.stocks;\n\nimport bbejeck.model.StockTransaction;\nimport bbejeck.model.StockTransactionCollector;\nimport bbejeck.serializer.JsonDeserializer;\nimport bbejeck.serializer.JsonSerializer;\nimport org.apache.kafka.common.serialization.Serde;\nimport org.apache.kafka.common.serialization.Serdes;\nimport org.apache.kafka.common.serialization.StringDeserializer;\nimport org.apache.kafka.common.serialization.StringSerializer;\nimport org.apache.kafka.streams.KafkaStreams;\nimport org.apache.kafka.streams.KeyValue;\nimport org.apache.kafka.streams.StreamsConfig;\nimport org.apache.kafka.streams.kstream.KStream;\nimport org.apache.kafka.streams.kstream.KStreamBuilder;\nimport org.apache.kafka.streams.kstream.TimeWindows;\nimport org.apache.kafka.streams.kstream.Windowed;\nimport org.apache.kafka.streams.kstream.internals.WindowedDeserializer;\nimport org.apache.kafka.streams.kstream.internals.WindowedSerializer;\nimport org.apache.kafka.streams.processor.WallclockTimestampExtractor;\n\nimport java.util.Properties;\n\n/**\n * User: Bill Bejeck\n * Date: 3/9/16\n * Time: 9:21 PM\n */\npublic class StocksKafkaStreamsDriver {\n\n public static void main(String[] args) {\n\n StreamsConfig streamingConfig = new StreamsConfig(getProperties());\n\n JsonSerializer stockTransactionsSerializer = new JsonSerializer<>();\n JsonDeserializer stockTransactionsDeserializer = new JsonDeserializer<>(StockTransactionCollector.class);\n JsonDeserializer stockTxnDeserializer = new JsonDeserializer<>(StockTransaction.class);\n JsonSerializer stockTxnJsonSerializer = new JsonSerializer<>();\n Serde transactionSerde = Serdes.serdeFrom(stockTxnJsonSerializer,stockTxnDeserializer);\n StringSerializer stringSerializer = new StringSerializer();\n StringDeserializer stringDeserializer = new StringDeserializer();\n Serde stringSerde = Serdes.serdeFrom(stringSerializer,stringDeserializer);\n Serde collectorSerde = Serdes.serdeFrom(stockTransactionsSerializer,stockTransactionsDeserializer);\n WindowedSerializer windowedSerializer = new WindowedSerializer<>(stringSerializer);\n WindowedDeserializer windowedDeserializer = new WindowedDeserializer<>(stringDeserializer);\n Serde> windowedSerde = Serdes.serdeFrom(windowedSerializer,windowedDeserializer);\n\n KStreamBuilder kStreamBuilder = new KStreamBuilder();\n\n\n KStream transactionKStream = kStreamBuilder.stream(stringSerde,transactionSerde,\"stocks\");\n\n transactionKStream.map((k,v)-> new KeyValue<>(v.getSymbol(),v))\n .through(stringSerde, transactionSerde,\"stocks-out\")\n .groupBy((k,v) -> k, stringSerde, transactionSerde)\n .aggregate(StockTransactionCollector::new,\n (k, v, stockTransactionCollector) -> stockTransactionCollector.add(v),\n TimeWindows.of(10000),\n collectorSerde, \"stock-summaries\")\n .to(windowedSerde,collectorSerde,\"transaction-summary\");\n\n\n System.out.println(\"Starting StockStreams Example\");\n KafkaStreams kafkaStreams = new KafkaStreams(kStreamBuilder,streamingConfig);\n kafkaStreams.start();\n System.out.println(\"Now started StockStreams Example\");\n\n }\n\n private static Properties getProperties() {\n Properties props = new Properties();\n props.put(StreamsConfig.CLIENT_ID_CONFIG, \"Stocks-Streams-Processor\");\n props.put(\"group.id\", \"stock-streams\");\n props.put(StreamsConfig.APPLICATION_ID_CONFIG, \"stocks_streams_id\");\n props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, \"localhost:9092\");\n props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, \"localhost:2181\");\n props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 1);\n props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class);\n return props;\n }\n}\n" }, { "path": "src/main/java/bbejeck/streams/twitter/HoseBirdTester.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.streams.twitter;\n\nimport com.google.common.collect.Lists;\nimport com.twitter.hbc.ClientBuilder;\nimport com.twitter.hbc.core.Client;\nimport com.twitter.hbc.core.Constants;\nimport com.twitter.hbc.core.Hosts;\nimport com.twitter.hbc.core.HttpHosts;\nimport com.twitter.hbc.core.endpoint.StatusesFilterEndpoint;\nimport com.twitter.hbc.core.processor.StringDelimitedProcessor;\nimport com.twitter.hbc.httpclient.auth.Authentication;\nimport com.twitter.hbc.httpclient.auth.OAuth1;\n\nimport java.util.List;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.LinkedBlockingDeque;\n\n/**\n * User: Bill Bejeck\n * Date: 3/28/16\n * Time: 8:55 PM\n */\npublic class HoseBirdTester {\n\n public static void main(String[] args) throws InterruptedException {\n\n BlockingQueue msgQueue = new LinkedBlockingDeque<>();\n\n\n\n Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);\n StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();\n List terms = Lists.newArrayList(\"superman vs batman\",\"#supermanvsbatman\");\n hosebirdEndpoint.trackTerms(terms);\n\n\n\n\n Authentication hosebirdAuth = new OAuth1(\"18qydWMuiUohwCtQpp1MOFCFr\",\n \"YrYhYd09LKZLbhsKT1o4XcEPl6HiAoNykiOxYBq0dAB8t0vRCo\",\n \"16972669-KSvyDEMc7dussPfW6a9Ru65L4eWGj637ciHLHZLyn\",\n \"ky53NE6cbBvtNLopto7o9gVyHDejSB2kPsRhHGKEd1MrS\");\n\n\n ClientBuilder clientBuilder = new ClientBuilder();\n clientBuilder.name(\"bbejeck-hosebird\")\n .hosts(hosebirdHosts)\n .authentication(hosebirdAuth)\n .endpoint(hosebirdEndpoint)\n .processor(new StringDelimitedProcessor(msgQueue));\n\n Client hosebirdClient = clientBuilder.build();\n hosebirdClient.connect();\n\n for (int msgRead = 0; msgRead < 100; msgRead++) {\n String msg = msgQueue.take();\n System.out.println(msg);\n }\n\n hosebirdClient.stop();\n\n }\n}\n" }, { "path": "src/main/java/bbejeck/streams/twitter/TwitterDataSource.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.streams.twitter;\n\nimport com.twitter.hbc.ClientBuilder;\nimport com.twitter.hbc.core.Client;\nimport com.twitter.hbc.core.Constants;\nimport com.twitter.hbc.core.Hosts;\nimport com.twitter.hbc.core.HttpHosts;\nimport com.twitter.hbc.core.endpoint.StatusesFilterEndpoint;\nimport com.twitter.hbc.core.processor.StringDelimitedProcessor;\nimport com.twitter.hbc.httpclient.auth.Authentication;\nimport com.twitter.hbc.httpclient.auth.OAuth1;\nimport org.apache.kafka.clients.producer.KafkaProducer;\nimport org.apache.kafka.clients.producer.Producer;\nimport org.apache.kafka.clients.producer.ProducerRecord;\n\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.IOException;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Properties;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.LinkedBlockingQueue;\n\n\n\n/**\n * User: Bill Bejeck\n * Date: 4/20/16\n * Time: 10:52 PM\n */\npublic class TwitterDataSource {\n\n private Client streamSource = null;\n private Producer producer = null;\n\n public static void main(String[] args) throws Exception {\n TwitterDataSource twitterDataSource = new TwitterDataSource();\n twitterDataSource.run();\n }\n\n public void run() {\n\n File propsFile = new File(\"src/main/resources/twitter-app.properties\");\n if (!propsFile.exists()) {\n System.out.println(\"Need to have twitter-app.properties, is it still named .template\");\n System.exit(1);\n }\n\n Properties props = new Properties();\n try {\n props.load(new FileInputStream(propsFile));\n } catch (IOException e) {\n throw new RuntimeException(e);\n }\n\n\n\n try {\n BlockingQueue twitterStreamQueue = new LinkedBlockingQueue<>();\n streamSource = getTwitterClient(props, twitterStreamQueue);\n producer = getKafkaProducer();\n\n\n int maxMessages = 500000;\n int counter = 0;\n\n streamSource.connect();\n\n while (counter++ < maxMessages) {\n String twitterMessage = null;\n try {\n twitterMessage = twitterStreamQueue.take();\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n }\n ProducerRecord message = new ProducerRecord<>(\"twitterData\", twitterMessage);\n producer.send(message);\n }\n\n } finally {\n stop();\n }\n }\n\n public void stop() {\n if (streamSource != null) {\n streamSource.stop();\n System.out.println(\"twitter streams stopped\");\n }\n if (producer != null) {\n producer.close();\n System.out.println(\"kafka producer closed\");\n }\n }\n\n private Client getTwitterClient(Properties props, BlockingQueue messageQueue) {\n\n String clientName = props.getProperty(\"clientName\");\n String consumerKey = props.getProperty(\"consumerKey\");\n String consumerSecret = props.getProperty(\"consumerSecret\");\n String token = props.getProperty(\"token\");\n String tokenSecret = props.getProperty(\"tokenSecret\");\n List searchTerms = Arrays.asList(props.getProperty(\"searchTerms\").split(\",\"));\n\n Authentication authentication = new OAuth1(consumerKey,consumerSecret,token,tokenSecret);\n Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);\n StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();\n\n hosebirdEndpoint.trackTerms(searchTerms);\n\n ClientBuilder clientBuilder = new ClientBuilder();\n clientBuilder.name(clientName)\n .hosts(hosebirdHosts)\n .authentication(authentication)\n .endpoint(hosebirdEndpoint)\n .processor(new StringDelimitedProcessor(messageQueue));\n\n return clientBuilder.build();\n\n }\n\n private Producer getKafkaProducer() {\n Properties props = new Properties();\n props.put(\"bootstrap.servers\", \"localhost:9092\");\n props.put(\"acks\", \"all\");\n props.put(\"retries\", 0);\n props.put(\"batch.size\", 16384);\n props.put(\"linger.ms\", 1);\n props.put(\"buffer.memory\", 33554432);\n props.put(\"key.serializer\", \"org.apache.kafka.common.serialization.StringSerializer\");\n props.put(\"value.serializer\", \"org.apache.kafka.common.serialization.StringSerializer\");\n\n return new KafkaProducer<>(props);\n }\n}\n" }, { "path": "src/main/java/bbejeck/streams/twitter/TwitterKStreamNLPDriver.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.streams.twitter;\n\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\n\n/**\n * User: Bill Bejeck\n * Date: 4/27/16\n * Time: 8:46 PM\n */\npublic class TwitterKStreamNLPDriver {\n\n public static void main(String[] args) {\n\n System.out.println(\"Starting the KStreams Twitter analysis\");\n\n ExecutorService service = Executors.newFixedThreadPool(2);\n TwitterDataSource twitterDataSource = new TwitterDataSource();\n TwitterStreamsAnalyzer twitterStreamsAnalyzer = new TwitterStreamsAnalyzer();\n\n Runnable dataSourceRunner = twitterDataSource::run;\n Runnable streamsRunner = twitterStreamsAnalyzer::run;\n\n service.submit(dataSourceRunner);\n service.submit(streamsRunner);\n\n try {\n //run for 10 minutes\n Thread.sleep(600000);\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n } finally {\n System.out.println(\"shutting down now\");\n service.shutdownNow();\n twitterDataSource.stop();\n twitterStreamsAnalyzer.stop();\n System.out.println(\"all done....\");\n }\n\n }\n}\n" }, { "path": "src/main/java/bbejeck/streams/twitter/TwitterStreamsAnalyzer.java", "content": "/*\n * Copyright 2016 Bill Bejeck\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage bbejeck.streams.twitter;\n\nimport bbejeck.model.Tweet;\nimport bbejeck.nlp.Classifier;\nimport bbejeck.serializer.JsonDeserializer;\nimport bbejeck.serializer.JsonSerializer;\nimport org.apache.commons.lang3.StringUtils;\nimport org.apache.kafka.common.serialization.Serde;\nimport org.apache.kafka.common.serialization.Serdes;\nimport org.apache.kafka.streams.KafkaStreams;\nimport org.apache.kafka.streams.StreamsConfig;\nimport org.apache.kafka.streams.kstream.KStream;\nimport org.apache.kafka.streams.kstream.KStreamBuilder;\nimport org.apache.kafka.streams.kstream.KeyValueMapper;\nimport org.apache.kafka.streams.kstream.Predicate;\nimport org.apache.kafka.streams.processor.WallclockTimestampExtractor;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.util.Properties;\n\n/**\n * User: Bill Bejeck\n * Date: 4/21/16\n * Time: 8:51 PM\n */\n@SuppressWarnings(\"unchecked\")\npublic class TwitterStreamsAnalyzer {\n\n private KafkaStreams kafkaStreams;\n\n public static void main(String[] args) throws IOException {\n TwitterStreamsAnalyzer streamsAnalyzer = new TwitterStreamsAnalyzer();\n streamsAnalyzer.run();\n }\n\n public void run() {\n StreamsConfig streamsConfig = new StreamsConfig(getProperties());\n\n JsonSerializer tweetJsonSerializer = new JsonSerializer<>();\n JsonDeserializer tweetJsonDeserializer = new JsonDeserializer<>(Tweet.class);\n Serde tweetSerde = Serdes.serdeFrom(tweetJsonSerializer, tweetJsonDeserializer);\n\n KStreamBuilder kStreamBuilder = new KStreamBuilder();\n\n Classifier classifier = new Classifier();\n classifier.train(new File(\"src/main/resources/kafkaStreamsTwitterTrainingData_clean.csv\"));\n\n KeyValueMapper languageToKey = (k, v) ->\n StringUtils.isNotBlank(v.getText()) ? classifier.classify(v.getText()):\"unknown\";\n\n Predicate isEnglish = (k, v) -> k.equals(\"english\");\n Predicate isFrench = (k, v) -> k.equals(\"french\");\n Predicate isSpanish = (k, v) -> k.equals(\"spanish\");\n\n KStream tweetKStream = kStreamBuilder.stream(Serdes.String(), tweetSerde, \"twitterData\");\n\n KStream[] filteredStreams = tweetKStream.selectKey(languageToKey).branch(isEnglish, isFrench, isSpanish);\n\n filteredStreams[0].to(Serdes.String(), tweetSerde, \"english\");\n filteredStreams[1].to(Serdes.String(), tweetSerde, \"french\");\n filteredStreams[2].to(Serdes.String(), tweetSerde, \"spanish\");\n\n kafkaStreams = new KafkaStreams(kStreamBuilder, streamsConfig);\n System.out.println(\"Starting twitter analysis streams\");\n kafkaStreams.start();\n System.out.println(\"Started\");\n\n }\n\n public void stop() {\n kafkaStreams.close();\n System.out.println(\"KStreams stopped\");\n }\n\n\n private static Properties getProperties() {\n Properties props = new Properties();\n props.put(StreamsConfig.CLIENT_ID_CONFIG, \"Twitter-Streams-Analysis\");\n props.put(\"group.id\", \"twitter-streams\");\n props.put(StreamsConfig.APPLICATION_ID_CONFIG, \"twitter-streams-id\");\n props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, \"localhost:9092\");\n props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, \"localhost:2181\");\n props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 1);\n props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class);\n return props;\n }\n}\n" }, { "path": "src/main/resources/kafkaStreamsTwitterTrainingData_clean.csv", "content": "spanish#Hacia la mitad de la calle de la Universidad, entre los números 51 y 57,\nspanish#se ven cuatro hoteles que pueden citarse entre los más lindos de París.\nspanish#El primero pertenece al señor Pozzo di Borgo, el segundo al condeMailly, el tercero al duque de Choiseul y el último, que hace esquina ala calle Bellechasse, al barón de Sanglié.\nspanish#El aspecto de este edificio es noble. La puerta cochera da entrada a unpatio de honor cuidadosamente enarenado y tapizado de parrascentenarias. El pabellón del portero está a la izquierda, envuelto entreel follaje espeso de la hiedra, donde los gorriones y los huéspedes dela garita parlotean al unísono. En el fondo del patio, a la derecha, unaamplia escalinata resguardada por una marquesina, conduce al vestíbulo ya la gran escalera.\nspanish#La planta baja y el primer piso están ocupados por el barón únicamente,\nspanish#que disfruta sin compartirlo con nadie un vasto jardín, limitado porotros jardines, y poblado de urracas, mirlos y ardillas que van y vienende ése a los otros en completa libertad, como si se tratara dehabitantes de un bosque y no de ciudadanos de París.\nspanish#Las armas de los Sanglié, pintadas en negro, se descubren en todas lasparedes del vestíbulo. Son un jabalí de oro en un campo de gules. Elescudo tiene por soporte dos lebreles, y está rematado con el penacho debarón con esta leyenda: _Sang lié au Roy_[A].\nspanish#Como media docena de lebreles vivos, agrupados según su capricho, seaburren al pie de la escalera, mordisquean las verónicas floridas en losvasos del Japón o se tienden sobre la alfombra alargando la cabezaserpentina. Los lacayos, sentados en banquetas de Beauvais, cruzansolemnemente los brazos, como conviene a los criados de buena casa.\nspanish#El día 1.º de enero de 1853, hacia las nueve de la mañana, toda laservidumbre del hotel celebraba en el vestíbulo un congreso tumultuoso.\nspanish#El administrador del barón, el señor Anatolio, acababa de distribuirlesel aguinaldo. El mayordomo había recibido quinientos francos, el ayudade cámara doscientos cincuenta. El menos favorecido de todos, elmarmitón, contemplaba con una ternura inefable dos hermosos luises deoro completamente nuevos. Habría celosos en la asamblea, perodescontentos ni uno solo, y cada uno a su manera decía que da gustoservir a un amo rico y generoso.\nspanish#Los tales individuos formaban un grupo bastante pintoresco alrededor deuna de las bocas del calorífero. Los más madrugadores llevaban ya lagran librea; los otros vestían aún el chaleco con mangas que constituyeel uniforme de media gala de los criados.\nspanish#El ayuda de cámara iba vestido de negro completamente, con zapatillas deorillo; el jardinero parecía un aldeano endomingado; el cochero llevabachaqueta de tricot y sombrero galoneado; el portero un tahalí de oro yzuecos. Aquí y acullá se distinguía a lo largo de las paredes, unafusta, una almohaza, un encerador, escobas, plumeros y algo más cuyonombre ignoro.\nspanish#El señor dormía hasta mediodía, como quien ha pasado la noche en elclub, y por lo tanto tenían tiempo para empezar sus faenas. Por lopronto se entretenían en darle empleo al dinero y las ilusiones lesocupaban bastante. Los hombres todos son algo parientes de aquellalechera de la fábula.\nspanish#Con esto, y lo que ya tengo ahorradodecía el mayordomo, puedoredondear mi renta vitalicia. A Dios gracias no falta el pan, y los díasde la vejez los tendré asegurados.\nspanish#Como es usted solteroreplico el ayuda de cámara, no tiene quepensar en nadie. Pero yo tengo familia. Por eso pienso entregarle eldinero a ese buen señor que va a la Bolsa, y algo me producirá.\nspanish#Es una buena idea, señor Fernandodijo el marmitón. Cuando vayausted, llévele mis cuarenta francos.\nspanish#El ayuda de cámara se creyó obligado también a intervenir y exclamó entono de protección:\nspanish#¡Vaya con el joven! ¿Qué crees tú que se puede hacer con cuarentafrancos en la Bolsa?\nspanish#Buenorespondió el joven ahogando un suspiro, los llevaré a la Cajade ahorros.\nspanish#El cochero soltó una ruidosa carcajada y se dio unos puñetazos sobre elestómago gritando:\nspanish#Esta es mi caja de ahorros. Aquí es donde he colocado siempre misfondos, y a fe que no me ha ido mal. ¿Verdad, padre Altorf?\nspanish#El padre Altorf, suizo[B] de profesión, alsaciano de nacimiento, deelevada estatura, vigoroso, huesudo, de desarrollado vientre, ancho dehombros, de cabeza enorme y rubicundo como un hipopótamo, sonrió con elrabillo del ojo y produjo con la lengua un pequeño chasquido que eratodo un poema.\nspanish#El jardinero, delicada flor de la Normandía, hizo sonar el dinero en sumano y respondió al honorable preopinante:\nspanish#¡Vamos, no diga usted tonterías! lo que se ha bebido ya no se vuelve atener. Lo mejor que hay es esconder el dinero en una pared vieja o en unárbol hueco. ¡Los que así lo hagan no darán de comer al notario!\nspanish#La asamblea en pleno protestó de la ingenuidad de aquel buen hombre queenterraba en flor sus escudos, sin hacerlos producir. Quince o diez yseis exclamaciones se elevaron al mismo tiempo. Cada uno expuso suopinión, descubrió su secreto, cabalgó en su Clavileño. Cada uno hizosaltar las monedas en su bolsillo y acarició ardientemente lasesperanzas ciertas, la dicha contante y sonante que habían embolsado. Eloro mezclaba su aguda vocecita con aquel concierto de pasiones vulgares;\nspanish#y el choque de las piezas de veinte francos, más embriagador que losvapores del vino o el olor de la pólvora, emborrachaba a aquellos pobrescerebros y aceleraba los latidos de sus groseros corazones.\nspanish#En lo más fuerte del tumulto, se abrió una pequeña puerta que daba a laescalera, entre el piso bajo y el primero. Una mujer, con un harapientotraje negro, descendió vivamente los peldaños, atravesó el vestíbulo,\nspanish#abrió la puerta de vidrieras y desapareció en el patio.\nspanish#Todo esto pasó en un minuto y, no obstante, la sombría aparición sellevó el buen humor de todas aquellas gentes, que se levantaron a supaso con el más profundo respeto. Los gritos se detuvieron en susgargantas y el oro ya no volvió a sonar en sus bolsillos. La pobre mujerhabía dejado detrás de ella como una estela de silencio y de estupor.\nspanish#El primero que se repuso fue el ayuda de cámara, que era lo que se llamaun espíritu fuerte.\nspanish#¡Voto a...!exclamó. He creído ver pasar a la miseria en persona.\nspanish#e ha estropeado el año. Ya veréis cómo no vuelve a salirme nada bienhasta el día de San Silvestre. ¡Brrr! tengo frío en la espalda.\nspanish#¡Pobre mujer!dijo el mayordomo. Ha tenido cientos y miles y ya laveis ahora... ¿Quién creería que es una duquesa?\nspanish#Es que el vagabundo de su marido se lo ha comido todo.\nspanish#¡Un jugador!\nspanish#¡Un hombre que no piensa más que en comer!\nspanish#Un andariego que trota de la mañana a la noche, con sus piernas derocín.\nspanish#No es él el que me interesa: tiene lo que se merece.\nspanish#¿Se sabe algo de la señorita Germana?\nspanish#Su negra me ha dicho que cada día está peor. A cada golpe de tos llenaun pañuelo.\nspanish#¡Y sin una alfombra en su habitación! Esa niña no se curaría más queen un país templado, en Italia, por ejemplo.\nspanish#Será un ángel para Dios.\nspanish#Los que quedan son más dignos de compasión.\nspanish#¡No sé cómo se las arreglará la duquesa para salir de este atolladero.\nspanish#¡A todos debe! Ultimamente el panadero se ha negado a fiarles más.\nspanish#¿Cuánto deben de alquiler?\nspanish#Ochocientos francos; pero lo que me extraña es que siquiera el señorhaya visto el color de su dinero.\nspanish#Si yo fuese él, preferiría tener desalquilado el piso antes quepermitir que viviesen en él personas que deshonran la casa.\nspanish#¡No seas bestia! ¿Para qué arrastrar por el arroyo al duque de La Tourde Embleuse y a su familia? Esas miserias, para que lo sepas, son comolas llagas del barrio; todos nosotros tenemos interés en ocultarlas.\nspanish#¡Toma!dijo el marmitón, creo que tengo razón para burlarme. ¿Porqué no trabajan? Los duques son hombres como los demás.\nspanish#¡Muchacho!exclamó gravemente el mayordomo, estás diciendo cosasincoherentes. La prueba de que no son hombres como los demás, es que yo,\nspanish#tu superior, no sería ni barón durante una hora de mi vida. Además, laduquesa es una mujer sublime y hace cosas de las que ni tú ni yoseríamos capaces. ¿Tomarías tú caldo durante todo un año y en todas lascomidas?\nspanish#¡Caramba! ¡No me parece eso muy divertido!\nspanish#¡Pues bien! la duquesa pone el puchero a la lumbre cada dos días,\nspanish#porque a su marido no le gusta la sopa de vigilia. El señor se come sutapioca de caldo graso y un bistec y un par de chuletas, y la pobre ysanta mujer se conforma con los desperdicios. Es hermoso, ¿verdad?\ngerman#Bär, der große, alte, struppige Hofhund auf Hoël, saß auf der Türtreppeund schaute ernsthaft über den Hof. Es war ein kalter, klarerSpätwintertag, und der Schnee glitzerte im Sonnenschein. Am liebstenwäre Bär aber doch hineingegangen; denn es ließ sich nicht leugnen: wieer da saß, fror ihn grimmig an den Pfoten, und er hob abwechselnd balddie eine, bald die andre eine Weile von den Steinfliesen empor, um nichtdas Kribbeln in die Klauen zu bekommen.\ngerman#Aber er durfte seinen Posten nicht verlassen. Die Schweine und dieZiegen waren heute im Freien. Noch führten sie sich zwar alle ganzanständig auf; die Schweine gingen dort in der Sonne und rieben sich ander Ecke des Kuhstalls, und weiter weg knabberten die Ziegen eifrig ander Baumrinde, die beim Schweinestall auf einen großen Haufen für siezusammengekehrt war, und taten so, als hätten sie an nichts andres zudenken. Aber er wußte von früher her: kaum war er hineingegangen, dalagerten sie sich sogleich mitten in den Haustüren und verübten all denUnfug, den sie sich nur ausdenken konnten die große neue ZiegeKrummhorn, die erst im letzten Herbst auf den Hof gekommen war, und dieer noch nicht ordentlich abgerichtet, hatte bereits einen Schwuppdichbis an die Hausecke hin gemacht und ihn dabei so gleichgültig undüberlegen angesehen.\ngerman#Die war wirklich eine unerträgliche Person, aber sie sollte sich bloß\ngerman#unterstehen !\ngerman#Ein Weilchen wenigstens mußte er noch sitzen bleiben die Wege durfteer ja auch nicht ganz aus dem Auge verlieren, es hätte doch jemandkommen können.\ngerman#Zufällig drehte er den Kopf nach dem schmalen Pfad hin, der die Haldeschräg vom Oberdorf herabkam.\ngerman#Alle Wetter! Was war denn das?\ngerman#Dort kam Etwas etwas Rundes, Putziges, Winziges ärgerlich, daß dieAugen nicht mehr recht mitwollten! ja, ja, er mußte auf alle FälleMeldung machen.\ngerman#Er fing an zu bellen, ein kurzes, tiefes Gekläff, das weithinausschallte. Die Ziegen sprangen ängstlich in einen Klumpen zusammenund spitzten die Ohren, die Schweine hörten jählings auf, sich zu juckenund zu kratzen, und lauschten ja, da konnte man sehen, daß sie vorihm Furcht hatten.\ngerman#Dann blieb er wieder ruhig sitzen und sah den Weg hinauf. Nein aber, ober jemals etwas Ähnliches gesehen hatte vielleicht war es nichteinmal etwas, das er zu melden brauchte; aber immerhin mußte er sichwohl auf den Weg machen und sich die Sache etwas näher ansehen.\ngerman#Er krümmte den buschigen Schwanz in einen großen Bogen; man solltesehen, daß er bester Laune war, und trippelte zum Hofe hinaus.\ngerman#Es mußte aber doch wohl ein Mensch sein. Es fing an, so leibhaftig derFinn-Kathrine zu gleichen, die dort im Winter zu gehen pflegte, aber diekonnte es doch nicht sein; denn dazu war das Wesen dort allzu winzig.\ngerman#Aber ein weiter, langer Weiberrock war es jedenfalls, und unter dem Rockkamen die Spitzen von einem Paar großer Schuhe hervor, über die graue,\ngerman#abgeschnittne Strumpffüße gezogen waren. Über den Rock war ein großerBausch Gestricktes gewurstelt, aus dem zwei Stummel mit roten,\ngerman#gestrickten Fausthandschuhen hervorguckten. Oben drauf saß ein etwaskleinerer Bausch Gestricktes das war wohl der Kopf. Hinten auf demRücken hing ein großes Bündel in einem dunkelfarbigen Einschlagetuch undvorn ein kleiner, niedlicher, rotgemalter Holzeimer.\ngerman#Bär mußte unwillkürlich stehen bleiben und sehen. Das rätselhafte Wesenwar nun ebenfalls seiner gewahr geworden und wie unschlüssig stehengeblieben. Da ging er auf die äußerste Wegkante hinüber, blieb dortstehen und versuchte, so gleichgültig wie möglich auszusehen, um dasWesen nicht zu erschrecken. Dies ging dann vorsichtig, wie auf Stelzen,\ngerman#langsam wieder vorwärts, indem es sich dicht an der andern Seite desWeges hindrückte und drehte sich allmählich, je näher es herankam, sodaß es ganz der Quere ging, als es endlich gerade vor Bär angelangt war.\ngerman#Da gelang es aber Bär, einen kurzen Blick durch eine kleine Öffnung indem obersten Strickbausch zu werfen, und was sah er! Erst ein kleines,\ngerman#rotes, aufwärtsstrebendes Stumpfnäschen, dann einen roten Mund, derunsicher zuckte, als wollte er zu weinen anfangen, und ein Paar großeblaue Augen, die ihn erschrocken anstarrten.\ngerman#Bah! Das war ja bloß ein kleines Mädel, das wegen der Kälte tüchtigeingemummelt war. Er kannte sie zwar nicht, aber wart mal dasEimerchen kam ihm so bekannt vor. Jedenfalls war es keine Art, sich hierbarsch zu stellen und so ein kleines Ding zu erschrecken.\ngerman#Unwillkürlich wedelte er mit dem Schwanze während er hinüberging, um denEimer zu beschnüffeln.\ngerman#Aber das kleine Mädchen verstand ihn nicht sofort, erschrocken trat esvielmehr ein paar Schritte zurück und purzelte rücklings neben dem Wegehin. Da sprang Bär rasch zur Seite und lief ein Stückchen voraus, sahsich wieder um und blinzelte freundlich mit den Augen und wedeltekräftig mit dem Schwanze. Jetzt begriff sie, stand auf, lächelte undtrippelte hinter ihm drein. Bär humpelte voran, sich immer wiederumsehend; nun erkannte er, daß sie sicher irgend einen Auftrag auf Hoëlauszurichten hatte, und da war es seine einfache Pflicht undSchuldigkeit, ihr zurecht zu helfen.\ngerman#Das kleine Mädchen war Sidsel Langröckchen von Schloß Guckaus oben aufder Höhe, die dergestalt auf Hoël ihren Einzug hielt.\ngerman#Schloß Guckaus lag auf einem öden, unfruchtbaren Berghang, weit vorn,\ngerman#gerade unter dem Großhammer, zu alleroberst im Oberdorf, und der Name \ngerman#es hieß eigentlich Neu-Wüstenland war ein Spitzname, den ihm einSpottvogel gegeben, weil man von da oben einen weiten Ausguck hatte, undweil es allem andern eher als einem Schloß glich. Das Krongut, das zumSchloß gehörte, bestand bloß aus etwas Heideland, wo Heidel- undPreiselbeerkraut üppig gedieh, unterbrochen hier und da von einemkleinen Fleckchen Ackerboden oder einem Stückchen Wiese.\ngerman#Die Stallgebäude bestanden aus einem untermauerten Kuhstall mit zweiStänden, halb in den Hügel eingegraben, und einem kleinen Schweinekofenim gleichen Stil. Und das Schloß selbst war ein winzig kleines, mitRasen gedecktes Häuschen, das ganz vorn am Abhang mitten in der Einödelag. Es hatte bloß ein niedriges Fensterchen mit ganz kleinen Scheiben,\ngerman#das ins Tal hinabschaute.\ngerman#Fast überall aber, wo man im Umkreis sein mochte wenn man in derRichtung hinsah und den Blick hoch genug hinaufwandte, überall sah manstets dieses Schloß und dies Guckfensterchen, das wie ein kleines Augeüber das Tal hinausblickte.\ngerman#Wenn nun die Herrlichkeit, von der sie herkam, nicht größer war, so kannman sich wohl leicht denken, daß Sidsel Langröckchen just keineverkleidete Prinzessin war, sondern schlecht und recht ein kleines armesBettelkind. Und zum ersten Mal auf den Hof von Hoël zu kommen, war fürsie dasselbe, als wenn sie wirklich zu Hofe gekommen wäre, obschon siein einem gar wichtigen, eigentlich nur für Erwachsene passenden Auftraggeschickt war; sie kam nämlich an Stelle ihrer Mutter als Spinnfrau.\ngerman#Sidsels Mutter, Rönnaug, hatte nun schon vier Jahre lang oben auf Schloß\ngerman#Guckaus allein für den Unterhalt der Familie sorgen müssen. Früher wares ihnen gut gegangen; da war aber der Mann gestorben, und nun saß sieallein da mit dem Schloß, einer Kuh und zwei Kindern, Jakob, der damalsungefähr sechs Jahre zählte, und Sidsel, die zwei Jahre jünger war. Eshielt oft schwer genug, aber sie hatten doch immerhin ein Dach über demKopfe, und nach Brennholz brauchten sie auch nicht weit zu laufen, derWald lag gerade vor der Tür.\ngerman#Im Sommer konnte sie den harten, steinigen Boden gerade soweitaufkratzen, daß sie auf dem jämmerlich kleinen Fleckchen AckerlandKartoffeln und etwas Korn bauen konnte, und Heidegras und frisches Laub,\ngerman#das sie sammelte, gab es gerade genug, um die Kuh Bliros jedes Jahrdurchfüttern zu können. Und wo eine Kuh ist, da gibt's auch immer was zuleben.\ngerman#Im Winter spann sie fleißig Leinwand und Wolle für die Bäuerinnen untenim Dorfe und vor allem für Kjersti Hoël, die Großbäuerin, bei der sieals Magd gedient hatte, ehe sie sich verheiratete.\ngerman#Auf diese Weise hatte sie sich durchgeschlagen. Unterdessen war derJakob so groß geworden, daß er selber für sich sorgen konnte. Im letztenFrühling war auf Nordrum Nachfrage nach einem Hirtenbuben gewesen, undda hatte er sofort zugeschlagen. Er und Sidsel hatten so oft oben in derStube vor dem kleinen Guckfensterchen auf den Knieen gelegen,\ngerman#hinausgeschaut und überlegt, wo sie wohl beide einmal dienen würden,\ngerman#wenn sie erst groß wären. Und da hatte der Knabe immer Nordrum für sichgewählt, hauptsächlich deshalb, weil er den Bauer von Nordrum immer alseinen besonders starken Mann hat rühmen hören, sie dagegen hattegemeint, besser müsse es auf Hoël sein, wo bloß Frauensleute wären.\ngerman#Im Herbst hatte dann der Nordrum gesagt, so einen Burschen wie den Jakobkönne er auch im Winter brauchen, und da war Jakob dort geblieben.\ngerman#Er war sogar schon letzte Weihnachten einen ganzen Tag wieder zu Hausegewesen, und da hatte er der Schwester ein Weihnachtsgeschenkmitgebracht von einem kleinen Mädchen auf dem Nordrumhofe, einen garfeinen Unterrock aus grauem Fries.\ngerman#Und wie lustig und spaßig er geworden war! Als sie den neuen Rock, derihr vorn wie hinten bis ganz herunter auf die Füße reichte, zumerstenmal angezogen hatte, da hatte er sie Sidsel _Langröckchen_\ngerman#genannt.\ngerman#Nach Weihnachten aber war oben auf Schloß Guckaus die Not eingezogen.\ngerman#Die Kuh Bliros, die sonst fast das ganze Jahr hindurch Milch gab, ließ\ngerman#es sich plötzlich einfallen, mehrere Monate trocken zu stehen; siesollte erst zum Sommer hin kalben. Die letzte Woche hatte es nichteinmal mehr Milch zum Kaffee gegeben.\ngerman#Bis zum Nachbarhof Svehaugen war es auch nicht bloß ein Katzensprung,\ngerman#und dort war es zudem auch knapp mit der Milch, das wußte Rönnaug, undaußerdem hatte sie keine Zeit, sie mußte sich sputen, daß sie mit derWolle, die sie für Kjersti Hoël spann, endlich fertig wurde und sie baldabliefern konnte, dann wurde wohl auch Rat für Milch und Kaffee undandres mehr. Deshalb arbeitete sie unausgesetzt die ganze Woche hindurch Sidsel war nun so groß, daß sie beim Karden helfen konnte undtrank den Kaffee schwarz. War es nun aber, weil sie den schwarzen Kaffeenicht vertragen konnte, oder ein andrer Grund, als sie gestern abendspät fertig geworden war, fühlte sie einen saugenden Schmerz unter derBrust, und als sie heute früh aufstand und sich fertig machte, mit derWolle nach Hoël zu gehen, wurde ihr mit einem Mal so übel undschwindlig, daß sie sich wieder aufs Bett legen mußte. Sie fühlte sichganz elend. Nun war es aber Sitte, daß die Spinnfrau, was siegesponnnen, auch selber brachte, und da bekam sie nicht bloß Vergütungfür ihre Arbeit, sondern wurde auch bewirtet und erhielt neue Aufträgeund Bescheid, wie das nächste Garn gesponnen werden sollte.\ngerman#Doch diesmal war wirklich kein andrer Ausweg, sie mußte Sidsel schicken.\ngerman#Sie würde sich schon zurechtfinden, obwohl sie noch nie auf Hoël gewesenwar, und soviel würde sie wohl auch mit nach Hause bringen, daß siewenigstens wieder einmal eine ordentliche Tasse Kaffee trinken könnten,\ngerman#dann konnte sie ja an einem der nächsten Tage immer noch selber gehen.\ngerman#Wenn sie sich nur darauf verlassen könnte, daß Sidsel sich ordentlich zubenehmen verstände und sich nicht gar zu ungelenk anstellte?\ngerman#O ja, hatte Sidsel gemeint, wenn sie nur gehen dürfte, dann würde siesich schon richtig zu benehmen wissen, genau wie eine Spinnfrau; dennsie erinnerte sich sehr gut daran, wie die es machten, von damals, alssie die Mutter nach Nordrum begleiten durfte.\ngerman#Da hatte denn Sidsel ihren Fries-Unterrock angezogen sie trug ihn wieeinen richtigen Rock und nur bei festlichen Gelegenheiten warordentlich in eine Menge Tücher und wollene Schale eingewickelt worden,\ngerman#hatte das große Garnknäuel hinten auf den Rücken gehängt bekommen, denMilcheimer vorn um den Hals und außerdem viele gute Lehren undErmahnungen mit auf den Weg, und so ging es zu, daß sie heute hinter Bärhertrippelnd in den Hof von Hoël einzog.\ngerman#Als sie an den Wirtschaftsgebäuden vorüberkam, mußte sie wirklich stehenbleiben und sich umsehen. Ja, hier war freilich alles viel großartigerals daheim! Ein Scheunentor so breit, daß das ganze Guckaus-Schloß hättehindurchfahren können, und eine einzige Fensterscheibe war mindestensebenso groß, wie das ganze Fenster oben im Schloß! Und solch eine Ziege!\ngerman# Sie bekam gerade Krummhorn zu Gesicht, die der Haustür bereitsbedenklich nahe war und kaum Miene machte auszureißen, als Bär herankam.\ngerman#Nicht einmal vor dem Hunde war die bange! Sie war freilich auch so groß\ngerman#wie ein ordentliches Kalb! Wenn die Kühe nach demselben Maßstabe waren,\ngerman#da mußten sie ja vom Erdboden aus das Gras auf dem Dache von Schloß\ngerman#Guckaus fressen können! Sie mußte unwillkürlich nach der Tür desKuhstalls hinschielen. Nein, die war doch nicht größer als Stalltürengewöhnlich sind da waren die Kühe doch wohl wie andre auch.\ngerman#Bär hatte unterdessen die Zeit benutzt, um Krummhorn zurecht zu weisen,\ngerman#nun kam er zurück, wedelte lustig mit dem Schwanze und wandte sich nachder Haustür, als ob er die Kleine hineingeleiten wollte. Ja, der Hundhatte recht, sie mußte sich wirklich beeilen, um ihren Auftragauszurichten und durfte nicht länger hier stehen und gaffen.\ngerman#Sie folgte Bär nach, ging in die Hausflur hinein, hob die Türklinke,\ngerman#drehte sich ganz um sich selbst herum, als sie die Tür wieder zumachte,\nfrench#Vers le milieu de la rue de l'Université, entre le numéro 51 et le 57,on voit quatre hôtels qui peuvent compter parmi les plus beaux deParis. Le premier appartient à M. Pozzo di Borgo; le second, au comtede Mailly; le troisième, au duc de Choiseul; le dernier au baron deSanglié. C'est celui qui fait l'angle de la rue Bellechasse.\nfrench#L'hôtel de Sanglié est une habitation de noble apparence. La portecochère s'ouvre sur une cour d'honneur soigneusement sablée et tapisséede treilles centenaires. La loge du suisse est à gauche, cachée sous unlierre épais où les moineaux et les portiers babillent à l'unisson. Aufond de la cour à droite, un large perron, abrité sous une marquise,conduit au vestibule et au grand escalier. Le rez-de-chaussée et lepremier sont occupés par le baron tout seul; il jouit sans partage d'unvaste jardin borné par d'autres jardins, peuplé de fauvettes, de merleset d'écureuils qui vont de l'un chez l'autre en pleine liberté, commes'ils étaient habitants d'un bois, et non citoyens de Paris.\nfrench#Les armes des Sanglié, peintes à la cire, se répètent sur tous les mursdu vestibule. C'est un sanglier d'or sur champ de gueules. L'écusson estsupporté par deux lévriers et surmonté d'un tortil de baron avec cettelégende: SANG LIÉ AU ROY. Une demi-douzaine de lévriers vivants, groupéssuivant leur fantaisie, s'agacent au pied de l'escalier, mordillent lesvéroniques en fleur dans les vases du Japon, ou s'aplatissent sur letapis en allongeant leur tête serpentine. Les valets de pied, assis surdes banquettes de Beauvais, se croisent solennellement les bras, commeil convient à des gens de bonne maison.\nfrench#Le 1er janvier 1853, vers les neuf heures du matin, tous les domestiquesde l'hôtel tenaient sous le vestibule un congrès tumultueux. L'intendantdu baron, M. Anatole, venait de leur distribuer leurs étrennes. Lemaître d'hôtel avait reçu cinq cents francs, le valet de chambre deuxcent cinquante. Le moins favorisé de tous, le marmiton, contemplait avecune tendresse inexprimable deux beaux louis d'or tout neufs. Il y avaitdes jaloux dans l'assemblée, mais pas un mécontent, et chacun disait enson langage que c'est plaisir de servir un maître riche et généreux.\nfrench#Ces messieurs formaient un groupe assez pittoresque autour d'une desbouches du calorifère. Les plus matineux avaient déjà la grande livrée;\nfrench#les autres portaient encore le gilet à manches, qui est la petite tenuedes domestiques. Le valet de chambre était tout de noir habillé, avecdes chaussons de lisière; le jardinier ressemblait à un villageoisendimanché; le cocher était en veste de tricot et en chapeau galonné; lesuisse, en baudrier d'or et en sabots. On apercevait ça et là, le longdes murs, un fouet, une étrille, un bâton à cirer, une tête de loup, etdes plumeaux dont je ne sais pas le nombre.\nfrench#Le maître dormait jusqu'à midi, en homme qui a passé la nuit au club:\nfrench#on avait bien le temps de se mettre à l'ouvrage. Chacun faisait d'avanceemploi de son argent, et les châteaux en Espagne allaient bon train.\nfrench#Tous les hommes, petits et grands, sont de la famille de Perrette quiportait un pot au lait.\nfrench#«Avec ça et ce que j'ai de côté, disait le maître d'hôtel, j'arrondiraima rente viagère. On a du pain sur la planche, Dieu merci! et l'on ne selaissera manquer de rien sur ses vieux jours.\nfrench#Parbleu! reprit le valet de chambre, vous êtes garçon; vous n'avezque vous à penser. Mais, moi, j'ai de la famille. Aussi, je donneraimon argent à ce petit jeune homme qui va à la Bourse. Il me tripoteraquelque chose.\nfrench#C'est une idée, ça, monsieur Ferdinand, repartit le marmiton.\nfrench#Portez-lui donc mes quarante francs, quand vous irez.»\nfrench#Le valet de chambre répondit d'un ton protecteur: «Est-il jeune!\nfrench#Qu'est-ce qu'on peut faire à la Bourse avec quarante francs?\nfrench#Allons, dit le jeune homme en étouffant un soupir, je les mettrai à lacaisse d'épargne!»\nfrench#Le cocher partit d'un gros éclat de rire. Il frappa sur son estomacen criant: «Ma caisse d'épargne, à moi, la voici. C'est là que j'aitoujours placé mes fonds, et je m'en suis bien trouvé. Pas vrai, pèreAltroff?»\nfrench#Le père Altroff, suisse de profession, Alsacien de naissance, grand,vigoureux, ossu, pansu, large des épaules, énorme de la tête, et aussirubicond qu'un jeune hippopotame, sourit du coin de l'oeil et fit avecsa langue un petit bruit qui valait un long poème.\nfrench#Le jardinier, fine fleur de Normand, fit sonner son argent dans sa main,et répondit à l'honorable préopinant: «Allais, marchais! ce qu'on a bu,on ne l'a plus. Il n'est tel placement qu'une bonne cachette dans unvieux mur ou dans un arbre creux. Argent bien enfouie, les notaires nela mangent point!»\nfrench#L'assemblée se récria sur la naïveté du bonhomme qui enterrait ses écustout vifs, au lieu de les faire travailler. Quinze ou seize exclamationss'élevèrent en même temps. Chacun dit son mot, trahit son secret,enfourcha son dada, secoua sa marotte. Chacun frappa sur sa poche etcaressa bruyamment les espérances certaines, le bonheur clair et liquidequ'il avait emboursé le matin. L'or mêlait sa petite voix aiguë à ceconcert de passions vulgaires; et le cliquetis des pièces de vingtfrancs, plus capiteux que la fumée du vin ou l'odeur de la poudre,enivrait ces pauvres cervelles et accélérait le battement de ces coeursgrossiers.\nfrench#Au plus fort du tumulte, une petite porte s'ouvrit sur l'escalier, entrele rez-de-chaussée et le premier étage. Une femme, vêtue de haillonsnoirs, descendit vivement les degrés, traversa le vestibule, ouvrit laporte vitrée et disparut dans la cour.\nfrench#Ce fut l'affaire d'une minute, et pourtant cette sombre apparitionéteignit la joie de tous ces valets en belle humeur. Ils se levèrent surson passage avec les marques d'un profond respect. Les cris s'arrêtèrentdans leur gosier, et l'or ne sonna plus dans leurs poches. La pauvrefemme avait laissé derrière elle comme une traînée de silence et destupeur.\nfrench#Le premier qui se remit fut le valet de chambre, un esprit fort.\nfrench#«Sapristi! cria-t-il, j'ai cru voir passer la misère en personne. Voilà\nfrench#mon jour de l'an gâté dès le matin. Vous verrez que rien ne me réussirajusqu'à la Saint-Sylvestre. Brrr! j'ai froid dans le dos.\nfrench#Pauvre femme! dit le maître d'hôtel. Ça a eu des mille et des cents,et puis voilà! Qui est-ce qui croirait que c'est une duchesse?\nfrench#C'est son gueux de mari qui lui a tout mangé.\nfrench#Un joueur!\nfrench#Un homme sur sa bouche!\nfrench#Un coureur qui trotte du matin au soir, avec ses vieilles jambes, à lasuite de tous les cotillons!\nfrench#C'est pas lui qui m'intéresse: il n'a que ce qu'il mérite.\nfrench#Sait-on comment va Mlle Germaine?\nfrench#Leur négresse m'a dit qu'elle était au plus bas. Elle crache le sang à\nfrench#plein mouchoir.\nfrench#Et pas de tapis dans sa chambre! Cette enfant-là ne guérirait que dansles pays chauds, à Florence ou en Italie.\nfrench#Ça fera un ange au ciel du bon Dieu.\nfrench#C'est ceux qui restent qui sont à plaindre!\nfrench#Je ne sais pas comment la duchesse sortira de là. Des comptes à\nfrench#n'en plus finir chez tous les fournisseurs! Le boulanger parle de leurrefuser crédit.\nfrench#Combien ont-ils de loyer là-haut?\nfrench#Huit cents. Mais je m'étonne si monsieur à jamais vu la couleur deleur argent.\nfrench#Si j'étais de lui, j'aimerais mieux laisser le petit appartementvacant que de garder des personnes qui font tache dans l'hôtel.\nfrench#Es-tu bête! Pour qu'on ramasse sur le pavé le duc de La Tourd'Embleuse et sa famille? Ces misères-là, vois-tu, c'est comme lesplaies du faubourg: nous avons tous intérêt à les cacher.\nfrench#Tiens! dit le marmiton, je m'en moque pas mal! Pourquoi qu'ils netravaillent pas? Les ducs sont des hommes comme les autres.\nfrench#Garçon! reprit gravement le maître d'hôtel, tu dis des chosesincohérentes. La preuve qu'ils ne sont pas des hommes comme les autres,c'est que moi, ton supérieur, je ne serai pas seulement baron pendantune heure de ma vie. D'ailleurs la duchesse est une femme sublime,et elle fait des choses dont ni toi ni moi ne serions capables.\nfrench#Mangerais-tu du bouilli pendant un an à tous tes repas?\nfrench#Dame! ça n'est pas amusant, le bouilli!\nfrench#Eh bien! la duchesse met le pot-au-feu tous les deux jours, parce queson mari n'aime pas la soupe maigre. Monsieur dîne d'un bon tapioca augras, avec un bifteck ou une paire de côtelettes, et la pauvre saintefemme avale jusqu'au dernier morceau de gîte qui se bouillit dans lamaison. Est-ce beau, cela?»\nfrench#Le marmiton fut touché dans l'âme. «Mon bon monsieur Tournoy, dit-ilau maître d'hôtel, c'est des gens bien intéressants. Est-ce qu'on nepourrait pas leur faire passer quelques douceurs, en s'entendant avecleur négresse?\nfrench#Ah bien oui! elle est aussi fière qu'eux; elle ne voudrait rien denous. Et cependant m'est avis qu'elle ne déjeune pas tous les jours.»\nfrench#Cette conversation aurait pu durer longtemps, si M. Anatole n'était venul'interrompre. Il entra juste à point pour couper la parole au chasseur,qui ouvrait la bouche pour la première fois. L'assemblée se dispersa entoute hâte; chaque orateur emporta ses instruments de travail, et il neresta dans la salle des délibérations qu'un de ces balais gigantesquesqu'on appelle tête de loup.\nfrench#Cependant Marguerite de Bisson, duchesse de La Tour d'Embleuse,cheminait à pas pressés dans la direction de la rue Jacob. Les passantsqui la frôlèrent du coude en courant donner ou recevoir des étrennes latrouvèrent semblable à ces Irlandaises désespérées qui piétinent sur lemacadam des rues de Londres à la poursuite d'un penny. Fille des ducsde Bretagne, femme d'un ancien gouverneur du Sénégal, la duchesseétait coiffée d'un chapeau de paille teinte en noir, dont les brides setordaient comme des ficelles. Une voilette d'imitation, percée en cinqou six endroits, cachait mal son visage et lui donnait une physionomieétrange. Cette belle tête, marquée de taches blanches d'inégalegrandeur, semblait défigurée par la petite vérole. Un vieux crêpe deChine, noirci par les soins du teinturier et roussi par les intempériesde l'air, laissait tomber tristement ses trois pointes, dont la frangeeffleurait la neige du trottoir. La robe qui se cachait là-dessous étaitsi fatiguée que le tissu était méconnaissable. Il eût fallu l'examinerde bien près et à la loupe pour reconnaître une moire ancienne démoirée,limée, coupée dans les plis, effrangée par en bas, et dévorée parla boue corrosive du pavé de Paris. Les souliers qui supportaient celamentable édifice n'avaient plus ni forme ni couleur. Le linge ne semontrait nulle part, ni au col, ni aux manches. Quelquefois, au passaged'un ruisseau, la robe se relevait à droite et laissait voir un bas delaine grise, un simple jupon de futaine noire. Les mains de la duchesse,rougies par un froid piquant, se cachaient sous son châle. Elle traînaitles pieds en marchant, non par une habitude de nonchalance, mais dans lapeur de perdre ses souliers.\nfrench#Par un contraste que vous avez pu observer quelquefois, la duchessen'était ni maigre, ni pâle, ni enlaidie en aucune façon par la misère.\nfrench#Elle avait reçu de ses ancêtres une de ces beautés rebelles quirésistent à tout, même à la faim. On a vu des prisonniers quiengraissaient dans leur cachot jusqu'à l'heure de la mort. A l'âge dequarante-sept ans, Mme de La Tour d'Embleuse conservait de beaux restesde jeunesse. Ses cheveux étaient noirs, et elle avait trente-deux dentscapables de broyer le pain le plus dur. Sa santé était moins florissanteque sa figure, mais c'est un secret qui restait entre elle et sonmédecin. La duchesse touchait à cette heure dangereuse et quelquefoismortelle où la femme disparaît pour faire place à l'aïeule. Plus d'unefois elle avait été saisie par des suffocations étranges. Elle rêvaitsouvent que le sang la prenait à la gorge pour l'étouffer. Des chaleursinexplicables lui montaient au cerveau par bouffées, et elle s'éveillaitdans un bain de vapeur animale où elle s'étonnait de ne point mourir. Ledocteur Le Bris, un jeune médecin et un vieil ami, lui recommandait unrégime doux, sans fatigues et surtout sans émotions. Mais quelle âmestoïcienne aurait traversé sans s'émouvoir de si rudes épreuves?\nfrench#Le duc César de La Tour d'Embleuse, fils d'un des émigrés les plusfidèles au roi et les plus acharnés contre le pays, fut récompensé\nfrench#magnifiquement des services de son père. En 1827, Charles X le nommagouverneur général de nos possessions dans l'Afrique occidentale. Ilétait à peine âgé de quarante ans. Pendant vingt-huit mois de séjourdans la colonie, il tint tête aux Maures et à la fièvre jaune; puis ildemanda un congé pour venir se marier à Paris. Il était riche, grâceau milliard d'indemnité; il doubla sa fortune en épousant la belleMarguerite de Bisson, qui possédait à Saint-Brieuc soixante mille livresde rente. Le roi signa son contrat le même jour que les ordonnances, etle duc se trouva marié et destitué tout d'un coup. Le nouveau pouvoirl'aurait accueilli volontiers dans la foule des transfuges; on dit mêmeque le ministère de Casimir Périer lui fit quelques avances. Il dédaignatous les emplois, par fierté d'abord, et autant par une invincibleparesse. Soit qu'il eût dépensé en trois ans tout ce qu'il avaitd'énergie, soit que la vie facile de Paris le retint par un attraitirrésistible, son seul travail pendant dix ans fut de promener seschevaux au Bois et de montrer ses gants jaunes au foyer de l'Opéra.\nenglish#The house stood on a slight rise just on the edge of the village. It stood on\nenglish#its own and looked over a broad spread of West Country farmland. Not a\nenglish#remarkable house by any means – it was about thirty years old, squattish,\nenglish#squarish, made of brick, and had four windows set in the front of a size and\nenglish#proportion which more or less exactly failed to please the eye.\nenglish#The only person for whom the house was in any way special was Arthur\nenglish#Dent, and that was only because it happened to be the one he lived in. He\nenglish#had lived in it for about three years, ever since he had moved out of London\nenglish#because it made him nervous and irritable. He was about thirty as well,\nenglish#dark haired and never quite at ease with himself. The thing that used to\nenglish#worry him most was the fact that people always used to ask him what he\nenglish#was looking so worried about. He worked in local radio which he always used\nenglish#to tell his friends was a lot more interesting than they probably thought. It\nenglish#was, too – most of his friends worked in advertising.\nenglish#On Wednesday night it had rained very heavily, the lane was wet and\nenglish#muddy, but the Thursday morning sun was bright and clear as it shone on\nenglish#Arthur Dent’s house for what was to be the last time\nenglish#It hadn’t properly registered with Arthur that the council wanted to\nenglish#knock down his house and build an bypass instead.\nenglish#At eight o’clock on Thursday morning Arthur didn’t feel very good. He\nenglish#woke up blearily, got up, wandered blearily round his room, opened a window,\nenglish#saw a bulldozer, found his slippers, and stomped off to the bathroom to wash.\nenglish#Toothpaste on the brush – so. Scrub.\nenglish#Shaving mirror – pointing at the ceiling. He adjusted it. For a moment\nenglish#it reflected a second bulldozer through the bathroom window. Properly adjusted,\nenglish#it reflected Arthur Dent’s bristles. He shaved them off, washed, dried,\nenglish#and stomped off to the kitchen to find something pleasant to put in his mouth.\nenglish#Kettle, plug, fridge, milk, coffee. Yawn.\nenglish#The word bulldozer wandered through his mind for a moment in search\nenglish#of something to connect with.\nenglish#The bulldozer outside the kitchen window was quite a big one.\nenglish#He stared at it. ”Yellow,” he thought and stomped off back to his bedroom\nenglish#to get dressed.\nenglish#Passing the bathroom he stopped to drink a large glass of water, and\nenglish#another. He began to suspect that he was hung over. Why was he hung\nenglish#over? Had he been drinking the night before? He supposed that he must\nenglish#have been. He caught a glint in the shaving mirror. ”Yellow,” he thought\nenglish#and stomped on to the bedroom.\nenglish#He stood and thought. The pub, he thought. Oh dear, the pub. He\nenglish#vaguely remembered being angry, angry about something that seemed important.\nenglish#He’d been telling people about it, telling people about it at great\nenglish#length, he rather suspected: his clearest visual recollection was of glazed\nenglish#looks on other people’s faces. Something about a new bypass he had just\nenglish#found out about. It had been in the pipeline for months only no one seemed\nenglish#to have known about it. Ridiculous. He took a swig of water. It would sort\nenglish#itself out, he’d decided, no one wanted a bypass, the council didn’t have a\nenglish#leg to stand on. It would sort itself out.\nenglish#God what a terrible hangover it had earned him though. He looked\nenglish#at himself in the wardrobe mirror. He stuck out his tongue. ”Yellow,” he\nenglish#thought. The word yellow wandered through his mind in search of something\nenglish#to connect with.\nenglish#Fifteen seconds later he was out of the house and lying in front of a big\nenglish#yellow bulldozer that was advancing up his garden path.\nenglish#Mr. L. Prosser was, as they say, only human. In other words he was a\nenglish#carbon-based life form descended from an ape. More specifically he was forty,\nenglish#fat and shabby and worked for the local council. Curiously enough, though\nenglish#he didn’t know it, he was also a direct male-line descendant of Genghis Khan,\nenglish#though intervening generations and racial mixing had so juggled his genes\nenglish#that he had no discernible Mongoloid characteristics, and the only vestiges\nenglish#left in Mr. L. Prosser of his mighty ancestry were a pronounced stoutness\nenglish#about the tum and a predilection for little fur hats.\nenglish#He was by no means a great warrior: in fact he was a nervous worried\nenglish#man. Today he was particularly nervous and worried because something had\nenglish#gone seriously wrong with his job – which was to see that Arthur Dent’s\nenglish#house got cleared out of the way before the day was out.\nenglish#”Come off it, Mr. Dent,”, he said, ”you can’t win you know. You can’t lie\nenglish#in front of the bulldozer indefinitely.” He tried to make his eyes blaze fiercely\nenglish#but they just wouldn’t do it.\nenglish#Arthur lay in the mud and squelched at him.\n" }, { "path": "src/main/resources/log4j.properties", "content": "#\n# Copyright 2016 Bill Bejeck\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n\nlog4j.appender.stdout=org.apache.log4j.ConsoleAppender\nlog4j.appender.stdout.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n\n\nlog4j.appender.file=org.apache.log4j.RollingFileAppender\nlog4j.appender.file.File=kafka_streams.log\nlog4j.appender.file.MaxFileSize=10MB\nlog4j.appender.file.MaxBackupIndex=2\nlog4j.appender.file.layout=org.apache.log4j.PatternLayout\nlog4j.appender.file.layout.ConversionPattern=[%d] %p %m (%c:%L)%n\n\n#log4j.logger.org.apache.kafka=INFO, file, stdout\n#log4j.logger.org.apache.kafka.clients.consumer.internals=INFO, file, stdout\n#log4j.logger.org.apache.kafka.common.metrics=INFO\n#log4j.logger.org.apache.kafka.streams.kstream.internals.KTableAggregate=INFO\n#log4j.logger.org.apache.kafka.streams.kstream=DEBUG\n#log4j.logger.org.apache.kafka.streams.processor.internals.StreamThread=INFO, stdout\n#log4j.logger.org.apache.kafka.streams.processor.internals.StreamTask=INFO, stdout\n#log4j.logger.org.apache.kafka.streams.processor.TopologyBuilder=INFO, stdout\n#log4j.logger.org.apache.kafka.streams.processor.internals.RecordQueue=INFO, stdout\n\n# zkclient can be verbose, during debugging it is common to adjust is separately\n#log4j.logger.org.I0Itec.zkclient.ZkClient=ERROR, file, stdout\n#log4j.logger.org.apache.zookeeper=ERROR, file, stdout\n\n" }, { "path": "src/main/resources/twitter-app.properties.template", "content": "clientName=\nconsumerKey=\nconsumerSecret=\ntoken=\ntokenSecret=\nsearchTerms=" }, { "path": "streaming-workflows/purchases-config.json", "content": "{\n \"workflows\": [\n {\n \"workflowName\": \"purchases\",\n \"workflowFilename\": \"purchases.json\"\n }\n ],\n \"producers\": [\n {\n \"type\": \"kafka\",\n \"broker.server\": \"127.0.0.1\",\n \"broker.port\": 9092,\n \"topic\": \"src-topic\",\n \"flatten\" : false,\n \"sync\": false\n },\n {\n \"type\": \"logger\"\n }\n ]\n}" }, { "path": "streaming-workflows/purchases.json", "content": "{\n \"eventFrequency\": 400,\n \"varyEventFrequency\": true,\n \"repeatWorkflow\": true,\n \"timeBetweenRepeat\": 1500,\n \"varyRepeatFrequency\": true,\n \"steps\": [\n {\n \"config\": [\n {\n \"lastName\" : \"lastName()\",\n \"firstName\" : \"firstName()\",\n \"creditCardNumber\": \"random('4929-3813-3266-4295', '5370-4638-8881-3020','4916-4811-5814-8111','4916-4034-9269-8783','5299-1561-5689-1938','5293-8502-0071-3058') \",\n \"itemPurchased\": \"random('batteries','eggs','diapers','shampoo','shaving cream','doughnuts','beer')\",\n \"quantity\": \"integer(1,4)\",\n \"price\": \"double(3.95,14.99)\",\n \"purchaseDate\": \"date(\\\"2016/02/12T00:00:00\\\",\\\"2016/02/19T23:59:59\\\")\" ,\n \"zipCode\" : \"random('20841','20852','19971','10005','21842')\"\n }\n ],\n \"duration\": 0\n }\n ]\n}" }, { "path": "streaming-workflows/stock-transactions-config.json", "content": "{\n \"workflows\": [\n {\n \"workflowName\": \"stock-transactions\",\n \"workflowFilename\": \"stock-transactions.json\"\n }\n ],\n \"producers\": [\n {\n \"type\": \"kafka\",\n \"broker.server\": \"127.0.0.1\",\n \"broker.port\": 9092,\n \"topic\": \"stocks\",\n \"flatten\" :false,\n \"sync\": false\n },\n {\n \"type\": \"logger\"\n }\n ]\n}" }, { "path": "streaming-workflows/stock-transactions.json", "content": "{\n \"eventFrequency\": 400,\n \"varyEventFrequency\": true,\n \"repeatWorkflow\": true,\n \"timeBetweenRepeat\": 1500,\n \"varyRepeatFrequency\": true,\n \"steps\": [\n {\n \"config\": [\n {\n \"timestamp\": \"now()\",\n \"symbol\": \"random('GOOG','MSFT','AAPL','YHOO','TWTR')\",\n \"amount\": \"double(100.0,60000.0)\",\n \"shares\": \"integer(100,1000)\",\n \"type\": \"random('purchase','sell')\"\n }\n ],\n \"duration\": 0\n }\n ]\n}" } ]