Repository: ozlerhakan/mongolastic Branch: master Commit: e7456115f39c Files: 32 Total size: 62.5 KB Directory structure: gitextract_t9r72zye/ ├── .gitignore ├── .travis.yml ├── Contributors.adoc ├── Dockerfile ├── LICENSE ├── README.adoc ├── pom.xml └── src/ ├── main/ │ ├── java/ │ │ └── com/ │ │ └── kodcu/ │ │ ├── config/ │ │ │ ├── ElasticConfiguration.java │ │ │ ├── FileConfiguration.java │ │ │ ├── MongoConfiguration.java │ │ │ ├── YamlConfiguration.java │ │ │ └── structure/ │ │ │ ├── Auth.java │ │ │ ├── Elastic.java │ │ │ ├── Misc.java │ │ │ ├── Mongo.java │ │ │ └── Namespace.java │ │ ├── listener/ │ │ │ └── BulkProcessorListener.java │ │ ├── main/ │ │ │ └── Mongolastic.java │ │ ├── provider/ │ │ │ ├── ElasticToMongoProvider.java │ │ │ ├── MongoToElasticProvider.java │ │ │ └── Provider.java │ │ ├── service/ │ │ │ ├── BulkService.java │ │ │ ├── ElasticBulkService.java │ │ │ └── MongoBulkService.java │ │ └── util/ │ │ └── codecs/ │ │ ├── CustomDateCodec.java │ │ └── CustomLongCodec.java │ └── resources/ │ └── log4j2.properties └── test/ ├── java/ │ └── com/ │ └── kodcu/ │ └── test/ │ ├── TestMongoToElastic.java │ └── TestMongolasticQueries.java └── resources/ ├── conf1 ├── conf2 └── conf3 ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ !.gitignore *.iml target .idea *.log *.class /.classpath /.project /.settings dependency-reduced-pom.xml ================================================ FILE: .travis.yml ================================================ sudo: required notifications: email: false language: java jdk: - oraclejdk8 os: - linux services: - docker env: - MONGOLASTIC_FULL="ozlerhakan/mongolastic:1.4.4" script: mvn install before_install: - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.1.1.deb && sudo dpkg -i --force-confnew elasticsearch-6.1.1.deb - sudo service elasticsearch start - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6 - echo "deb http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.4.list - sudo apt-get update - sudo apt-get install -y mongodb-org - sudo service mongod start - mongod --version before_script: - wget https://dl.dropbox.com/s/jvngnitppao8hay/tweets.zip - unzip tweets.zip - sleep 15 - curl http://localhost:9200/ - mongorestore -h 127.0.0.1 --port 27017 dump - mvn clean after_success: - docker build -t $MONGOLASTIC_FULL . - docker tag $MONGOLASTIC_FULL ozlerhakan/mongolastic:1.4; - docker tag $MONGOLASTIC_FULL ozlerhakan/mongolastic:latest; - docker images - if [ "$TRAVIS_BRANCH" == "master" ]; then docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"; docker push $MONGOLASTIC_FULL; docker push ozlerhakan/mongolastic:1.4; docker push ozlerhakan/mongolastic:latest; fi deploy: provider: releases skip_cleanup: true api_key: secure: E8iX+y1svn/88ftT5fnMIfECNUNek0T/VtrkdacuFriC/cNcopcZUuIEqmCTuMIvo+iL/h4F4wYzGLadRU6iOhnPk8oMPadcR+2Pll/mz7/I4zcj/iZL8KMJGlJxLpnTKBw3QEGnWflg/S8I6g6bOcGnPhQW0AQZ6bj7FBR59Gd7abtqAkwZVki+zxXbMzMqA3VYRaMc2UEdTU6AlcR7GeCdcDFhQ/4uqmMySGGGKLAROZ+0r9BhbaZ0ivuR7Uza9l68d0FhN2wRxDhP86RM3hbZttGQw+CBw7+3IcHoJ9MpSGgatxWd3eudWypDoocVs9FxYKS3zJcVzzLWtmB1yRdY/zOcLkr7Tkz2z+LIZbjPJFjeDUgpRJPqw8ckPOEbsdmp+/0zVBp+aDdMgpa3hfXkchTa4hXUDWJCZiuN4oByWxnBqoALUAjRdIflCeZuukbmHzZb6QWNTfRSGAZtQHO+sdJIvMFFC9v0k6T2WXvlEWDXt0cA8Gid9iBZLZfemJiBRNRP94TDcakMZzwnBH/G3l1nc7tk8AR7gX0b/biLeloSdxdI7UGV+uLvy+LgrsJp5TeLvBaylCRJ6672IvroGZwZm/GG3NsPS1vb2WoZfbjiS+paWUxreeFumJyg43Vuv0LiDND2y2P8zTtnC7HaM0k54ewt86Kx5w86Ma0= file: /home/travis/build/ozlerhakan/mongolastic/target/mongolastic.jar on: tags: true all_branches: true repo: ozlerhakan/mongolastic ================================================ FILE: Contributors.adoc ================================================ == Contributors Thank you for our contributors to make this project more usable! ✨ * https://github.com/hakdogan[Hüseyin Hakdoğan] * https://github.com/winder[Will Winder] * https://github.com/wareninja[Yılmaz Güleryüz] ================================================ FILE: Dockerfile ================================================ FROM openjdk:8-jdk-alpine MAINTAINER Hakan Ozler ADD target/mongolastic.jar . ENTRYPOINT ["java","-jar","mongolastic.jar","-f"] ================================================ FILE: LICENSE ================================================ The MIT License (MIT) Copyright (c) 2015-2016 Kodcu.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.adoc ================================================ = Mongolastic :version: v1.4.3 image:https://travis-ci.org/ozlerhakan/mongolastic.svg?branch=master["Build Status", link="https://travis-ci.org/ozlerhakan/mongolastic"] image:https://api.codacy.com/project/badge/Grade/8d768c2fc37246019115e4d090a33b98["Codacy code quality", link="https://www.codacy.com/app/ozlerhakan/mongolastic?utm_source=github.com&utm_medium=referral&utm_content=ozlerhakan/mongolastic&utm_campaign=Badge_Grade"] image:https://img.shields.io/docker/pulls/ozlerhakan/mongolastic.svg["Docker Pulls,link="https://hub.docker.com/r/ozlerhakan/mongolastic"] image:https://img.shields.io/github/release/ozlerhakan/mongolastic.svg[] image:https://img.shields.io/badge/mongo.java.driver-3.4.2-brightgreen.svg[] image:https://img.shields.io/badge/elastic.java.driver-6.2.4-brightgreen.svg[] image:https://img.shields.io/badge/license-MIT-blue.svg[] Mongolastic enables you to migrate your datasets from a mongod node to an elasticsearch node and vice versa. Since mongo and elastic servers can run with different characteristics, the tool provides several optional and required features to ably connect them. Mongolastic works with a yaml or json configuration file to begin a migration process. It reads your demand on the file and start syncing data in the specified direction. == How it works First, you can either pull the corresponding image of the app from https://hub.docker.com/r/ozlerhakan/mongolastic/[Docker Hub] Supported tags and respective Dockerfile links: * `_1.4_`, `_1.4.4_`, `_latest_` https://github.com/ozlerhakan/mongolastic/blob/master/Dockerfile[_(master/Dockerfile)_] * `_1.4.3_` https://github.com/ozlerhakan/mongolastic/blob/0dacd80cbdf7b5b7b282bf6dd89ede8558021577/Dockerfile[_(1.4.2/Dockerfile)_] or download the latest https://github.com/ozlerhakan/mongolastic/releases/download/{version}/mongolastic.jar[mongolastic.jar] file. Second, create a yaml or json file which must contain the following structure: [source,yaml] ---- misc: dindex: name: <1> as: <2> ctype: name: <3> as: <4> direction: (em | me) <5> batch: <6> dropDataset: <7> mongo: host: <8> port: <9> query: "mongo-query" <10> project: "projection" <11> auth: <12> user: pwd: "password" source: mechanism: ( plain | scram-sha-1 | x509 | gssapi | cr ) elastic: host: <13> port: <14> dateFormat: "" <15> longToString: <16> clusterName: <17> auth: <18> user: pwd: "password" ---- <1> the _database/index name_ to connect to. <2> another _database/index name_ in which documents will be located in the target service (*Optional*) <3> the _collection/type name_ to export. <4> another _collection/type name_ in which indexed/collected documents will reside in the target service (*Optional*) <5> direction of the data transfer. the default direction is me (that is, mongo to elasticsearch). You can skip this option if your data move from mongo to es. <6> Override the default batch size which is normally 200. (*Optional*) <7> configures whether or not the target table should be dropped prior to loading data. Default value is true (*Optional*) <8> the name of the host machine where the `mongod` is running. <9> the port where the `mongod` instance is listening. <10> data will be transferred based on a json mongodb query (*Optional*) <11> with 1.4.1, you can manipulate documents that will be migrated from mongo to es based on the https://docs.mongodb.com/manual/reference/operator/aggregation/project/[`$project`] operator (*Optional*) <12> as of v1.3.5, you can access an auth mongodb by giving auth configuration. (*Optional*) <13> the name of the host machine where the `elastic node` is running. <14> the *transport* port where the transport module will communicate with the running elastic node. E.g. *9300* for node-to-node communication. <15> a custom formatter for Date fields rather than the default DateCodec (*Optional*) <16> serialize long value as a string for backwards compatibility with other tools (*Optional*) <17> connect to a spesific elastic cluster (*Optional*) <18> as of v1.3.9, you can access an auth elastic search by giving auth configuration. (*Optional*) --- Alternatively, a JSON file can be specified as a mongolastic configuration file including the same YAML file structure above. [source,json] ---- { "misc": { "dindex": { "name": "twitter", "as": "media" }, "ctype": { "name": "tweets", "as": "posts" }, "direction": "me", "batch": 400, "dropDataset": true }, "mongo": { "host": "127.0.0.1", "port": 27017, "query": "{ lang: 'en' }", "project": "{ user:1, name:'$user.name', location: { $substr: [ '$user.location', 10, 15 ] }}", "auth": { "user": "joe", "pwd": "1234", "source": "twitter", "mechanism": "scram-sha-1" } }, "elastic": { "host": "127.0.0.1", "port": 9300, "dateFormat": "yyyy-MM-dd", "longToString": true, "auth": { "user": "joe", "pwd": "4321" } } } ---- == Example #1 The following files have the same configuration details: .yaml file [source,yaml] ---- misc: dindex: name: twitter as: kodcu ctype: name: tweets as: posts mongo: host: localhost port: 27017 query: "{ 'user.name' : 'kodcu.com'}" elastic: host: localhost port: 9300 ---- .json file [source,json] ---- { "misc": { "dindex": { "name": "twitter", "as": "kodcu" }, "ctype": { "name": "tweets", "as": "posts" } }, "mongo": { "host": "localhost", "port": 27017, "query": "{ 'user.name' : 'kodcu.com'}" }, "elastic": { "host": "localhost", "port": 9300 } } ---- the config says that the transfer direction is from mongodb to elasticsearch, mongolastic first looks at the _tweets_ collection, where the _user name_ is _kodcu.com_, of the _twitter_ database located on a mongod server running on default host interface and port number. If It finds the corresponding data, It will start copying those into an elasticsearch environment running on default host and transport number. After all, you should see a type called _"posts"_ in an index called _"kodcu"_ in the current elastic node. Why the index and type are different is because "dindex.as" and "ctype.as" options were set, these indicates that your data being transferred exist in _posts_ type of the _kodcu_ index. After downloading the jar or pulling the image and providing a conf file, you can either run the tool as: $ java -jar mongolastic.jar -f config.file __or__ $ docker run --rm -v $(PWD)/config.file:/config.file --net host ozlerhakan/mongolastic: config.file == Example #2 Using the project field, you are able to manipulate documents when migrating them from mongodb to elasticsearch. For more examples about the `$project` operator of the aggregation pipeline, take a look at its https://docs.mongodb.com/manual/reference/operator/aggregation/project/[documentation]. [source,yaml] ---- misc: dindex: name: twitter ctype: name: tweets mongo: host: 192.168.10.151 port: 27017 project: "{ user: 1, name: '$user.name', location: { $substr: [ '$user.location', 10, 15 ] }}" <1> elastic: host: 192.168.10.152 port: 9300 ---- <1> the migrated documents will include the user field and contain new fields `name` and `location`. NOTE: Every attempt of running the tool drops the mentioned db/index in the target environment unless the dropDataset parameter is configured otherwise. == License Mongolastic is released under http://showalicense.com/?hide_explanations=false&year=2015&fullname=Kodcu.com#license-mit[MIT]. ================================================ FILE: pom.xml ================================================ 4.0.0 com.kodcu.mongolastic mongolastic 1.4.4 UTF-8 1.8 6.2.4 3.4.2 2.8 2015 mongolastic A tool that migrates data from mongodb to elasticsearch and vice versa. http://github.com/ozlerhakan/mongolastic Kodcu.com ozlerhakan Hakan Ozler ozler.hakan@gmail.com http://github.com/ozlerhakan developer hakdogan Hüseyin Akdoğan huseyin.akdogan@kodcu.com http://github.com/hakdogan software evangelist - developer 3.0 github.com https://github.com/ozlerhakan/mongolastic/issues elasticsearch-releases https://artifacts.elastic.co/maven true false org.mongodb mongodb-driver ${mongo.version} org.slf4j slf4j-log4j12 log4j log4j org.elasticsearch.client transport ${elastic.version} org.slf4j slf4j-log4j12 log4j log4j org.apache.logging.log4j log4j-slf4j-impl ${logger.version} org.apache.logging.log4j log4j-api ${logger.version} org.apache.logging.log4j log4j-core ${logger.version} com.fasterxml.jackson.core jackson-databind 2.8.6 junit junit 4.12 test ${project.artifactId} org.apache.maven.plugins maven-shade-plugin 3.0.0 package shade com.kodcu.main.Mongolastic *:* META-INF/*.SF META-INF/*.DSA META-INF/*.RSA org.apache.maven.plugins maven-compiler-plugin 3.6.1 ${java.version} ${java.version} ================================================ FILE: src/main/java/com/kodcu/config/ElasticConfiguration.java ================================================ package com.kodcu.config; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Objects; import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings.Builder; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.transport.client.PreBuiltTransportClient; /** * Created by hakdogan on 21/05/15. */ public class ElasticConfiguration { private final Logger logger = LogManager.getLogger(ElasticConfiguration.class); private final YamlConfiguration config; private Client client; public ElasticConfiguration(final YamlConfiguration config) { this.config = config; this.prepareClient(); } private void prepareClient() { Builder settingsBuilder = applySettings(); try { TransportAddress ista = new TransportAddress(InetAddress.getByName(config.getElastic().getHost()), config.getElastic().getPort()); client = new PreBuiltTransportClient(settingsBuilder.build()) .addTransportAddress(ista); } catch (UnknownHostException ex) { logger.error(ex.getMessage(), ex); System.exit(-1); } } private Builder applySettings() { Builder settingsBuilder = Settings.builder(); settingsBuilder.put("client.transport.ping_timeout", "15s"); settingsBuilder.put("client.transport.nodes_sampler_interval", "5s"); // YG: to ensure reliable connection & resolve NoNodeAvailableException settingsBuilder.put("client.transport.sniff", true); settingsBuilder.put("network.bind_host", 0); // YG: for supporting ES Auth with ES Shield Optional.ofNullable(config.getElastic().getAuth()) .ifPresent(auth -> settingsBuilder.put("xpack.security.user", String.join(":", auth.getUser(), auth.getPwd()))); if (Objects.nonNull(config.getElastic().getClusterName())) { settingsBuilder.put("cluster.name", config.getElastic().getClusterName()); } else { settingsBuilder.put("client.transport.ignore_cluster_name", true); } return settingsBuilder; } public void closeNode() { client.close(); } public Client getClient() { return client; } } ================================================ FILE: src/main/java/com/kodcu/config/FileConfiguration.java ================================================ package com.kodcu.config; import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.Objects; /** * Created by Hakan on 5/19/2015. */ public class FileConfiguration { private final Logger logger = LoggerFactory.getLogger(FileConfiguration.class); private final String parameter; public FileConfiguration(String parameter) { this.parameter = parameter; } public YamlConfiguration getFileContent() { YamlConfiguration config = null; File ymlFile = new File(parameter); try { Yaml yaml = new Yaml(); if (ymlFile.isFile()) { FileInputStream configFile = new FileInputStream(ymlFile); config = yaml.loadAs(configFile, YamlConfiguration.class); } else { // we expect that this is just a string including yaml format config = yaml.loadAs(parameter, YamlConfiguration.class); } } catch (Exception e) { try { ObjectMapper mapper = new ObjectMapper(); config = mapper.readValue(ymlFile, YamlConfiguration.class); } catch (IOException ex) { logger.error(e.getMessage(), e); System.exit(0); } } logger.info(System.lineSeparator() + "Config Output:" + System.lineSeparator() + config.toString() + System.lineSeparator()); config = this.controlAsSettings(config); return config; } private YamlConfiguration controlAsSettings(YamlConfiguration config) { String dIndexAs = config.getMisc().getDindex().getAs(); String cTypeAs = config.getMisc().getCtype().getAs(); if (Objects.isNull(dIndexAs)) config.getMisc().getDindex().setAs(config.getMisc().getDindex().getName()); if (Objects.isNull(cTypeAs)) config.getMisc().getCtype().setAs(config.getMisc().getCtype().getName()); if (config.getMisc().getBatch() < 200) config.getMisc().setBatch(200); return config; } } ================================================ FILE: src/main/java/com/kodcu/config/MongoConfiguration.java ================================================ package com.kodcu.config; import com.mongodb.MongoClient; import com.mongodb.MongoClientOptions; import com.mongodb.MongoCredential; import com.mongodb.ReadPreference; import com.mongodb.ServerAddress; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.Objects; import static com.mongodb.assertions.Assertions.notNull; /** * Created by Hakan on 5/19/2015. */ public class MongoConfiguration { private final Logger logger = LoggerFactory.getLogger(MongoConfiguration.class); private final YamlConfiguration config; private MongoClient client; public MongoConfiguration(final YamlConfiguration config) { this.config = config; this.prepareClient(); } private void prepareClient() { try { ServerAddress address = new ServerAddress(config.getMongo().getHost(), config.getMongo().getPort()); MongoClientOptions options = MongoClientOptions.builder() .serverSelectionTimeout(5000) .socketKeepAlive(false) .readPreference(ReadPreference.primaryPreferred()) .sslInvalidHostNameAllowed(true) .build(); client = connectToClient(address, options); } catch (Exception ex) { logger.error(ex.getMessage(), ex); System.exit(-1); } } private MongoClient connectToClient(ServerAddress address, MongoClientOptions options) { if (Objects.nonNull(config.getMongo().getAuth())) { String user = notNull("auth.name", config.getMongo().getAuth().getUser()); String database = config.getMongo().getAuth().getSource(); char[] pwd = config.getMongo().getAuth().getPwd().toCharArray(); String mechanism = config.getMongo().getAuth().getMechanism(); MongoCredential credential = findMongoCredential(user, database, pwd, mechanism); return new MongoClient(Arrays.asList(address), Arrays.asList(credential), options); } else { return new MongoClient(Arrays.asList(address), options); } } private MongoCredential findMongoCredential(String user, String database, char[] pwd, String mechanism) { MongoCredential credential = null; switch (mechanism) { case "scram-sha-1": credential = MongoCredential.createScramSha1Credential(user, database, pwd); break; case "x509": credential = MongoCredential.createMongoX509Credential(user); break; case "cr": credential = MongoCredential.createMongoCRCredential(user, database, pwd); break; case "plain": credential = MongoCredential.createPlainCredential(user, database, pwd); break; case "gssapi": credential = MongoCredential.createGSSAPICredential(user); break; default: credential = MongoCredential.createCredential(user, database, pwd); break; } return credential; } public MongoCollection getMongoCollection() { MongoCollection collection = null; try { MongoDatabase database = client.getDatabase(config.getMisc().getDindex().getName()); collection = database.getCollection(config.getMisc().getCtype().getName()); } catch (Exception ex) { logger.error(ex.getMessage(), ex); } return collection; } public void closeConnection() { if (Objects.nonNull(client)) client.close(); } public MongoClient getClient() { return client; } } ================================================ FILE: src/main/java/com/kodcu/config/YamlConfiguration.java ================================================ package com.kodcu.config; import com.kodcu.config.structure.Elastic; import com.kodcu.config.structure.Misc; import com.kodcu.config.structure.Mongo; /** * Created by Hakan on 5/19/2015. */ public class YamlConfiguration { private Misc misc; private Mongo mongo; private Elastic elastic; public Elastic getElastic() { return elastic; } public void setElastic(Elastic elastic) { this.elastic = elastic; } public Misc getMisc() { return misc; } public void setMisc(Misc misc) { this.misc = misc; } public Mongo getMongo() { return mongo; } public void setMongo(Mongo mongo) { this.mongo = mongo; } @Override public String toString() { return "{" + "elastic=" + elastic + ", misc=" + misc + ", mongo=" + mongo + '}'; } } ================================================ FILE: src/main/java/com/kodcu/config/structure/Auth.java ================================================ package com.kodcu.config.structure; /** * Created by Hakan on 5/14/2016. */ public class Auth { private String user; private String pwd; private String source; private String mechanism; public String getUser() { return user; } public void setUser(String user) { this.user = user; } public String getPwd() { return pwd; } public void setPwd(String pwd) { this.pwd = pwd; } public String getSource() { return source; } public void setSource(String source) { this.source = source; } public String getMechanism() { return mechanism; } public void setMechanism(String mechanism) { this.mechanism = mechanism; } @Override public String toString() { return "Auth [user=" + user + ", pwd=" + pwd + ", source=" + source + ", mechanism=" + mechanism + "]"; } } ================================================ FILE: src/main/java/com/kodcu/config/structure/Elastic.java ================================================ package com.kodcu.config.structure; /** * Created by Hakan on 1/16/2016. */ public class Elastic { private String host; private int port; private String dateFormat; private Boolean longToString = false; private String clusterName; // optional private Auth auth; // optional public Auth getAuth() { return auth; } public void setAuth(Auth auth) { this.auth = auth; } public String getClusterName() { return clusterName; } public void setClusterName(String clusterName) { this.clusterName = clusterName; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public int getPort() { return port; } public void setPort(int port) { this.port = port; } public String getDateFormat() { return dateFormat; } public void setDateFormat(String dateFormat) { this.dateFormat = dateFormat; } public Boolean getLongToString() { return this.longToString; } public void setLongToString(Boolean longToString) { this.longToString = longToString; } @Override public String toString() { return "Elastic{" + "host='" + host + '\'' + ", port=" + port + ", clusterName=" + clusterName + ", dateFormat=" + dateFormat + ", longToString=" + longToString + ", auth=" + auth + '}'; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Elastic elastic = (Elastic) o; if (port != elastic.port) return false; return host != null ? host.equals(elastic.host) : elastic.host == null; } @Override public int hashCode() { int result = host != null ? host.hashCode() : 0; result = 31 * result + port; return result; } } ================================================ FILE: src/main/java/com/kodcu/config/structure/Misc.java ================================================ package com.kodcu.config.structure; /** * Created by Hakan on 1/16/2016. */ public class Misc { private String direction = "me"; private Namespace dindex; private Namespace ctype; private Boolean dropDataset = true; private int batch = 200; public int getBatch() { return batch; } public void setBatch(int batch) { this.batch = batch; } public Namespace getCtype() { return ctype; } public void setCtype(Namespace ctype) { this.ctype = ctype; } public String getDirection() { return direction; } public void setDirection(String direction) { this.direction = direction; } public Namespace getDindex() { return dindex; } public void setDindex(Namespace dindex) { this.dindex = dindex; } public Boolean getDropDataset() { return this.dropDataset; } public void setDropDataset(Boolean dropDataset) { this.dropDataset = dropDataset; } @Override public String toString() { return "Misc{" + "batch=" + batch + ", direction='" + direction + '\'' + ", dindex=" + dindex + ", ctype=" + ctype + ", dropDataset=" + dropDataset + '}'; } } ================================================ FILE: src/main/java/com/kodcu/config/structure/Mongo.java ================================================ package com.kodcu.config.structure; /** * Created by Hakan on 1/16/2016. */ public class Mongo { private String host; private int port; private String query = "{}"; private String project; private Auth auth; @Override public String toString() { return "Mongo{" + "host='" + host + '\'' + ", port=" + port + ", query='" + query + '\'' + ", project='" + project + '\'' + ", auth=" + auth + '}'; } public String getProject() { return project; } public void setProject(String project) { this.project = project; } public Auth getAuth() { return auth; } public void setAuth(Auth auth) { this.auth = auth; } public String getQuery() { return query; } public void setQuery(String query) { this.query = query; } public int getPort() { return port; } public void setPort(int port) { this.port = port; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } } ================================================ FILE: src/main/java/com/kodcu/config/structure/Namespace.java ================================================ package com.kodcu.config.structure; /** * Created by Hakan on 1/16/2016. */ public class Namespace { private String name; private String as; public String getAs() { return as; } public void setAs(String as) { this.as = as; } public String getName() { return name; } public void setName(String name) { this.name = name; } @Override public String toString() { return "Namespace{" + "as='" + as + '\'' + ", name='" + name + '\'' + '}'; } } ================================================ FILE: src/main/java/com/kodcu/listener/BulkProcessorListener.java ================================================ package com.kodcu.listener; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Created by Hakan on 5/21/2015. */ public class BulkProcessorListener implements BulkProcessor.Listener { private final Logger logger = LoggerFactory.getLogger(BulkProcessorListener.class); @Override public void beforeBulk(long executionId, BulkRequest request) { } @Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { if (response.hasFailures()) { logger.error(response.buildFailureMessage()); } else { logger.info(String.format("Data transfer successfully terminated.(%d)", response.getItems().length)); } } @Override public void afterBulk(long executionId, BulkRequest request, Throwable failure) { logger.error("Transfer failed."); logger.error(failure.getMessage(), failure.fillInStackTrace()); } } ================================================ FILE: src/main/java/com/kodcu/main/Mongolastic.java ================================================ package com.kodcu.main; import com.kodcu.config.ElasticConfiguration; import com.kodcu.config.FileConfiguration; import com.kodcu.config.MongoConfiguration; import com.kodcu.config.YamlConfiguration; import com.kodcu.provider.ElasticToMongoProvider; import com.kodcu.provider.MongoToElasticProvider; import com.kodcu.provider.Provider; import com.kodcu.service.BulkService; import com.kodcu.service.ElasticBulkService; import com.kodcu.service.MongoBulkService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Optional; /** * Created by Hakan on 5/19/2015. */ public class Mongolastic { private static final Logger logger = LoggerFactory.getLogger(Mongolastic.class); private final String parameter; public Mongolastic(String parameter) { this.parameter = parameter; } public static void main(String[] args) throws Exception { configAssertion(args); String parameter = args[1]; Mongolastic app = new Mongolastic(parameter); app.start(); } private static void configAssertion(String[] args) { if (args.length == 0) { logger.error("Incorrect syntax. Should be mongolastic.jar -f /path/file"); System.exit(0); } if (!args[0].equals("-f")) { logger.error("Please specify the -f parameter with a correct yaml or json file"); System.exit(0); } if (args.length != 2) { logger.error("Incorrect syntax. Pass max 2 parameters"); System.exit(0); } } public void start() { FileConfiguration fConfig = new FileConfiguration(parameter); Optional yamlConfig = Optional.ofNullable(fConfig.getFileContent()); long begin = System.currentTimeMillis(); try { yamlConfig.ifPresent(this::proceedService); } finally { logger.info("Load duration: " + (System.currentTimeMillis() - begin) + "ms"); } } public void proceedService(YamlConfiguration config) { ElasticConfiguration elastic = new ElasticConfiguration(config); MongoConfiguration mongo = new MongoConfiguration(config); BulkService bulkService = this.initializeBulkService(config, mongo, elastic); Provider provider = this.initializeProvider(config, mongo, elastic); provider.transfer(bulkService, config, () -> { bulkService.close(); mongo.closeConnection(); elastic.closeNode(); }); } private Provider initializeProvider(YamlConfiguration config, MongoConfiguration mongo, ElasticConfiguration elastic) { if (config.getMisc().getDirection().equals("em")) { return new ElasticToMongoProvider(elastic, config); } return new MongoToElasticProvider(mongo.getMongoCollection(), config); } private BulkService initializeBulkService(YamlConfiguration config, MongoConfiguration mongo, ElasticConfiguration elastic) { if (config.getMisc().getDirection().equals("em")) { return new MongoBulkService(mongo.getClient(), config); } return new ElasticBulkService(config, elastic); } } ================================================ FILE: src/main/java/com/kodcu/provider/ElasticToMongoProvider.java ================================================ package com.kodcu.provider; import com.kodcu.config.ElasticConfiguration; import com.kodcu.config.YamlConfiguration; import org.bson.Document; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.common.unit.TimeValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; /** * Created by Hakan on 6/29/2015. */ public class ElasticToMongoProvider implements Provider { private final Logger logger = LoggerFactory.getLogger(ElasticToMongoProvider.class); private final ElasticConfiguration elastic; private final YamlConfiguration config; private SearchResponse response; public ElasticToMongoProvider(final ElasticConfiguration elastic, final YamlConfiguration config) { this.elastic = elastic; this.config = config; } @Override public long getCount() { long count = 0; IndicesAdminClient admin = elastic.getClient().admin().indices(); IndicesExistsRequestBuilder builder = admin.prepareExists(config.getMisc().getDindex().getName()); if (builder.execute().actionGet().isExists()) { SearchResponse countResponse = elastic.getClient().prepareSearch(config.getMisc().getDindex().getName()) .setTypes(config.getMisc().getCtype().getName()) .setSearchType(SearchType.QUERY_THEN_FETCH) .setSize(0) .execute().actionGet(); count = countResponse.getHits().getTotalHits(); } else { logger.info("Index/Type does not exist or does not contain the record"); System.exit(-1); } logger.info("Elastic Index/Type count: " + count); return count; } @Override public List buildJSONContent(int skip, int limit) { if (Objects.isNull(response)) { response = elastic.getClient().prepareSearch(config.getMisc().getDindex().getName()) .setTypes(config.getMisc().getCtype().getName()) .setSearchType(SearchType.QUERY_THEN_FETCH) .setScroll(new TimeValue(60000)) .setSize(limit) .execute().actionGet(); } else { response = elastic.getClient() .prepareSearchScroll(response.getScrollId()) .setScroll(new TimeValue(60000)) .execute().actionGet(); } return Arrays.stream(response.getHits().getHits()) .map(hit -> new Document(hit.getSourceAsMap())) .collect(Collectors.toList()); } } ================================================ FILE: src/main/java/com/kodcu/provider/MongoToElasticProvider.java ================================================ package com.kodcu.provider; import com.kodcu.config.YamlConfiguration; import com.mongodb.client.AggregateIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import org.bson.Document; import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; import java.util.Optional; import static com.mongodb.client.model.Aggregates.*; /** * Created by Hakan on 5/18/2015. */ public class MongoToElasticProvider implements Provider { private final Logger logger = LoggerFactory.getLogger(MongoToElasticProvider.class); private final MongoCollection collection; private final YamlConfiguration config; private MongoCursor cursor; private long cursorId = 0; public MongoToElasticProvider(final MongoCollection collection, final YamlConfiguration config) { this.collection = collection; this.config = config; } @Override public long getCount() { long count = collection.count(Document.parse(config.getMongo().getQuery())); logger.info("Mongo collection count: " + count); if (count == 0) { logger.error("Database/Collection does not exist or does not contain the record"); System.exit(-1); } return count; } @Override public List buildJSONContent(int skip, int limit) { ArrayList result = new ArrayList<>(limit); result.ensureCapacity(limit); MongoCursor cursor = getCursor(skip); while (cursor.hasNext() && result.size() < limit) { result.add(cursor.next()); } return result; } /** * Get the MongoDB cursor. */ private MongoCursor getCursor(int skip) { if (cursor == null && cursorId == 0) { Document query = Document.parse(config.getMongo().getQuery()); List pipes = new ArrayList<>(3); pipes.add(match(query)); pipes.add(skip(skip)); Optional.ofNullable(config.getMongo().getProject()).ifPresent(p -> pipes.add(project(Document.parse(p)))); AggregateIterable aggregate = collection.aggregate(pipes) .allowDiskUse(true) .useCursor(true); cursor = aggregate.iterator(); // TODO: Persist cursor ID somewhere to allow restarts. Optional.ofNullable(cursor.getServerCursor()).ifPresent(serverCursor -> cursorId = serverCursor.getId()); } else if (cursor == null && cursorId != 0) { // TODO: Lookup cursor ID for resume. // Open existing cursor in case of restart?? } return cursor; } } ================================================ FILE: src/main/java/com/kodcu/provider/Provider.java ================================================ package com.kodcu.provider; import com.kodcu.config.YamlConfiguration; import com.kodcu.service.BulkService; import org.bson.Document; import java.util.List; /** * Created by Hakan on 6/30/2015. */ public interface Provider { default void transfer(final BulkService bulkService, final YamlConfiguration config, final Runnable closeConnections) { long count = this.getCount(); final int limit = config.getMisc().getBatch(); int skip = 0; if (count != 0 && config.getMisc().getDropDataset()) bulkService.dropDataSet(); while (count >= limit) { List content = this.buildJSONContent(skip, limit); bulkService.proceed(content); count -= limit; skip += limit; } if (count > 0) { List content = this.buildJSONContent(skip, (int) count); bulkService.proceed(content); } closeConnections.run(); } long getCount(); List buildJSONContent(int skip, int limit); } ================================================ FILE: src/main/java/com/kodcu/service/BulkService.java ================================================ package com.kodcu.service; import java.util.List; /** * Created by Hakan on 6/30/2015. */ public interface BulkService { void proceed(List content); void dropDataSet(); void close(); } ================================================ FILE: src/main/java/com/kodcu/service/ElasticBulkService.java ================================================ package com.kodcu.service; import com.kodcu.config.ElasticConfiguration; import com.kodcu.config.YamlConfiguration; import com.kodcu.listener.BulkProcessorListener; import com.kodcu.util.codecs.CustomDateCodec; import com.kodcu.util.codecs.CustomLongCodec; import com.mongodb.MongoClient; import org.bson.Document; import org.bson.codecs.BsonTypeClassMap; import org.bson.codecs.Codec; import org.bson.codecs.DocumentCodec; import org.bson.codecs.Encoder; import org.bson.codecs.configuration.CodecRegistries; import org.bson.codecs.configuration.CodecRegistry; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; /** * Created by Hakan on 5/21/2015. */ public class ElasticBulkService implements BulkService { private final Logger logger = LoggerFactory.getLogger(ElasticBulkService.class); private final YamlConfiguration config; private final ElasticConfiguration client; private final BulkProcessor bulkProcessor; private final Encoder encoder; public ElasticBulkService(final YamlConfiguration config, final ElasticConfiguration client) { this.config = config; this.client = client; this.bulkProcessor = BulkProcessor.builder(client.getClient(), new BulkProcessorListener()) .setBulkActions(config.getMisc().getBatch()) .setFlushInterval(TimeValue.timeValueSeconds(5)) .setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB)) .build(); encoder = getEncoder(); } @Override public void proceed(List content) { try { logger.info("Transferring data began to elasticsearch."); final String indexName = config.getMisc().getDindex().getAs(); final String typeName = config.getMisc().getCtype().getAs(); for (Object o : content) { Document doc = (Document) o; Object id = doc.get("_id"); IndexRequest indexRequest = new IndexRequest(indexName, typeName, String.valueOf(id)); doc.remove("_id"); indexRequest.source(doc.toJson(encoder), XContentType.JSON); bulkProcessor.add(indexRequest); } } catch (Exception ex) { logger.debug(ex.getMessage(), ex); } } @Override public void close() { try { bulkProcessor.awaitClose(10, TimeUnit.MINUTES); } catch (InterruptedException ex) { logger.error(ex.getMessage(), ex); } } @Override public void dropDataSet() { final String indexName = config.getMisc().getDindex().getAs(); IndicesAdminClient admin = client.getClient().admin().indices(); IndicesExistsRequestBuilder builder = admin.prepareExists(indexName); if (builder.execute().actionGet().isExists()) { DeleteIndexResponse delete = admin.delete(new DeleteIndexRequest(indexName)).actionGet(); if (delete.isAcknowledged()) logger.info(String.format("The current index %s was deleted.", indexName)); else logger.info(String.format("The current index %s was not deleted.", indexName)); } } /** * Customizations for the document.toJson output. *

* http://mongodb.github.io/mongo-java-driver/3.0/bson/codecs/ * * @return the toJson encoder. */ private Encoder getEncoder() { ArrayList> codecs = new ArrayList<>(); if (config.getElastic().getDateFormat() != null) { // Replace default DateCodec class to use the custom date formatter. codecs.add(new CustomDateCodec(config.getElastic().getDateFormat())); } if (config.getElastic().getLongToString()) { // Replace default LongCodec class codecs.add(new CustomLongCodec()); } if (codecs.size() > 0) { BsonTypeClassMap bsonTypeClassMap = new BsonTypeClassMap(); CodecRegistry codecRegistry = CodecRegistries.fromRegistries( CodecRegistries.fromCodecs(codecs), MongoClient.getDefaultCodecRegistry()); return new DocumentCodec(codecRegistry, bsonTypeClassMap); } else { return new DocumentCodec(); } } } ================================================ FILE: src/main/java/com/kodcu/service/MongoBulkService.java ================================================ package com.kodcu.service; import com.kodcu.config.YamlConfiguration; import com.mongodb.MongoClient; import com.mongodb.client.MongoCollection; import org.bson.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; /** * Created by Hakan on 6/29/2015. */ public class MongoBulkService implements BulkService { private final Logger logger = LoggerFactory.getLogger(MongoBulkService.class); private final MongoCollection collection; public MongoBulkService(final MongoClient client, final YamlConfiguration config) { this.collection = client.getDatabase(config.getMisc().getDindex().getAs()).getCollection(config.getMisc().getCtype().getAs()); } @Override public void proceed(List content) { try { logger.info("Transferring data began to mongodb."); collection.insertMany((List) content); } catch (Exception ex) { logger.debug(ex.getMessage(), ex); } } @Override public void dropDataSet() { if (collection.count() != 0) { String collectionName = collection.getNamespace().getCollectionName(); collection.drop(); logger.info(String.format("The current collection called %s was deleted.", collectionName)); } } @Override public void close() { //no-op } } ================================================ FILE: src/main/java/com/kodcu/util/codecs/CustomDateCodec.java ================================================ package com.kodcu.util.codecs; import org.bson.BsonWriter; import org.bson.codecs.DateCodec; import org.bson.codecs.EncoderContext; import java.text.SimpleDateFormat; import java.util.Date; /** * @author wwinder * Created on: 5/27/16 */ public class CustomDateCodec extends DateCodec { private final SimpleDateFormat formatter; public CustomDateCodec(String format) { formatter = new SimpleDateFormat(format); } @Override public void encode(final BsonWriter writer, final Date value, final EncoderContext encoderContext) { writer.writeString(formatter.format(value)); } } ================================================ FILE: src/main/java/com/kodcu/util/codecs/CustomLongCodec.java ================================================ package com.kodcu.util.codecs; import org.bson.BsonWriter; import org.bson.codecs.EncoderContext; import org.bson.codecs.LongCodec; /** * @author wwinder * Created on: 5/27/16 */ public class CustomLongCodec extends LongCodec { @Override public void encode(final BsonWriter writer, final Long value, final EncoderContext encoderContext) { writer.writeString(value.toString()); } } ================================================ FILE: src/main/resources/log4j2.properties ================================================ name=PropertiesConfig property.filename = mongolastic appenders = console, file appender.console.type = Console appender.console.name = STDOUT appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%-5level] [%d{yyyy-MM-dd HH:mm:ss}] [%t] [%p]: - %msg%n appender.file.type = File appender.file.name = LOGFILE appender.file.fileName=mongolastic.log appender.file.layout.type=PatternLayout appender.file.layout.pattern=[%-5level] [%d{yyyy-MM-dd HH:mm:ss}] [%t] [%p]: - %msg%n loggers=file logger.file.name=guru.springframework.blog.log4j2properties logger.file.level = debug logger.file.appenderRefs = file logger.file.appenderRef.file.ref = LOGFILE rootLogger.level = info rootLogger.appenderRefs = stdout rootLogger.appenderRef.stdout.ref = STDOUT ================================================ FILE: src/test/java/com/kodcu/test/TestMongoToElastic.java ================================================ package com.kodcu.test; import com.kodcu.config.ElasticConfiguration; import com.kodcu.config.FileConfiguration; import com.kodcu.config.MongoConfiguration; import com.kodcu.config.YamlConfiguration; import com.kodcu.provider.ElasticToMongoProvider; import com.kodcu.provider.MongoToElasticProvider; import com.kodcu.provider.Provider; import com.kodcu.service.BulkService; import com.kodcu.service.ElasticBulkService; import com.kodcu.service.MongoBulkService; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.IndicesAdminClient; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import java.util.Arrays; import java.util.Objects; import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; /** * Created by Hakan on 9/8/2015. */ @RunWith(Parameterized.class) public class TestMongoToElastic { private final FileConfiguration file; public TestMongoToElastic(final FileConfiguration file) { super(); this.file = file; } @Parameterized.Parameters(name = "{index}: ({0})={1}") public static Iterable queries() throws Exception { String query = "misc:\n" + " dindex:\n" + " name: twitter\n" + " ctype:\n" + " name: tweets\n" + " batch: 500\n" + "mongo:\n" + " host: mongo\n" + " port: 27017\n" + "elastic:\n" + " host: es\n" + " port: 9300"; return Arrays.asList(new Object[][]{ {new FileConfiguration("src/test/resources/conf1")}, {new FileConfiguration("src/test/resources/conf3")}, {new FileConfiguration(query)} }); } @Test public void shouldCopyOneQueryToEsFromMongoDB() { YamlConfiguration config = file.getFileContent(); assertThat(config, is(notNullValue())); if (Objects.isNull(config.getMongo().getAuth())) if (Objects.nonNull(System.getenv("MONGO_IP"))) config.getMongo().setHost(System.getenv("MONGO_IP")); else config.getMongo().setHost("localhost"); else { if (Objects.nonNull(System.getenv("MONGO_AUTH_IP"))) config.getMongo().setHost(System.getenv("MONGO_AUTH_IP")); else return; } if (Objects.isNull(System.getenv("ES_IP"))) config.getElastic().setHost("localhost"); else config.getElastic().setHost(System.getenv("ES_IP")); ElasticConfiguration elastic = new ElasticConfiguration(config); MongoConfiguration mongo = new MongoConfiguration(config); BulkService bulkService = this.initializeBulkService(config, mongo, elastic); assertThat(bulkService, is(instanceOf(ElasticBulkService.class))); Provider provider = this.initializeProvider(config, mongo, elastic); assertThat(provider, is(instanceOf(MongoToElasticProvider.class))); provider.transfer(bulkService, config, () -> { bulkService.close(); assertThat(provider.getCount(), equalTo(this.getCount(elastic, config))); elastic.closeNode(); mongo.closeConnection(); }); } private Provider initializeProvider(YamlConfiguration config, MongoConfiguration mongo, ElasticConfiguration elastic) { if (config.getMisc().getDirection().equals("em")) { return new ElasticToMongoProvider(elastic, config); } return new MongoToElasticProvider(mongo.getMongoCollection(), config); } private BulkService initializeBulkService(YamlConfiguration config, MongoConfiguration mongo, ElasticConfiguration elastic) { if (config.getMisc().getDirection().equals("em")) { return new MongoBulkService(mongo.getClient(), config); } return new ElasticBulkService(config, elastic); } public long getCount(ElasticConfiguration elastic, YamlConfiguration config) { IndicesAdminClient admin = elastic.getClient().admin().indices(); IndicesExistsRequestBuilder builder = admin.prepareExists(config.getMisc().getDindex().getAs()); assertThat(builder.execute().actionGet().isExists(), is(true)); elastic.getClient().admin().indices().flush(new FlushRequest(config.getMisc().getDindex().getAs())).actionGet(); SearchResponse response = elastic.getClient().prepareSearch(config.getMisc().getDindex().getAs()) .setTypes(config.getMisc().getCtype().getAs()) .setSearchType(SearchType.QUERY_THEN_FETCH) .setSize(0) .execute().actionGet(); long count = response.getHits().getTotalHits(); return count; } } ================================================ FILE: src/test/java/com/kodcu/test/TestMongolasticQueries.java ================================================ package com.kodcu.test; import com.kodcu.config.FileConfiguration; import com.kodcu.config.YamlConfiguration; import com.kodcu.config.structure.Elastic; import com.kodcu.config.structure.Misc; import com.kodcu.config.structure.Mongo; import com.kodcu.config.structure.Namespace; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import java.util.Arrays; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertThat; /** * Created by Hakan on 8/24/2015. */ @RunWith(Parameterized.class) public class TestMongolasticQueries { private final FileConfiguration config; private final YamlConfiguration expected; public TestMongolasticQueries(final FileConfiguration config, final YamlConfiguration expected) { super(); this.config = config; this.expected = expected; } @Parameterized.Parameters(name = "{index}: ({0})={1}") public static Iterable queries() throws Exception { return Arrays.asList(new Object[][]{ {new FileConfiguration("src/test/resources/conf1"), createQueryConfiguration1()}, {new FileConfiguration("src/test/resources/conf2"), createQueryConfiguration2()} }); } private static YamlConfiguration createQueryConfiguration1() { YamlConfiguration query = new YamlConfiguration(); Misc misc = new Misc(); Namespace db = new Namespace(); db.setName("twitter"); db.setAs("kodcu"); misc.setDindex(db); Namespace c = new Namespace(); c.setName("tweets"); c.setAs("tweets"); misc.setCtype(c); misc.setBatch(300); query.setMisc(misc); Mongo mongod = new Mongo(); mongod.setHost("mongo"); mongod.setPort(27017); mongod.setQuery("{}"); query.setMongo(mongod); Elastic es = new Elastic(); es.setHost("es"); es.setPort(9300); query.setElastic(es); return query; } private static YamlConfiguration createQueryConfiguration2() { YamlConfiguration query = new YamlConfiguration(); Misc misc = new Misc(); Namespace db = new Namespace(); db.setName("twitter"); db.setAs("twitter"); misc.setDindex(db); Namespace c = new Namespace(); c.setName("tweets"); c.setAs("posts"); misc.setCtype(c); misc.setDirection("em"); misc.setBatch(200); query.setMisc(misc); Mongo mongod = new Mongo(); mongod.setHost("127.0.0.1"); mongod.setPort(27017); query.setMongo(mongod); Elastic es = new Elastic(); es.setHost("127.0.0.1"); es.setPort(9300); query.setElastic(es); return query; } @Test public void shouldProceedQueries() { YamlConfiguration actual = config.getFileContent(); assertThat(actual, notNullValue()); assertThat(actual.getMongo().getQuery(), is(expected.getMongo().getQuery())); assertThat(actual.getMongo().getHost(), is(expected.getMongo().getHost())); assertThat(actual.getMongo().getPort(), is(expected.getMongo().getPort())); assertThat(actual.getElastic().getHost(), is(expected.getElastic().getHost())); assertThat(actual.getElastic().getPort(), is(expected.getElastic().getPort())); assertThat(actual.getMisc().getDindex().getName(), is(expected.getMisc().getDindex().getName())); assertThat(actual.getMisc().getDindex().getAs(), is(expected.getMisc().getDindex().getAs())); assertThat(actual.getMisc().getCtype().getName(), is(expected.getMisc().getCtype().getName())); assertThat(actual.getMisc().getCtype().getAs(), is(expected.getMisc().getCtype().getAs())); } } ================================================ FILE: src/test/resources/conf1 ================================================ misc: dindex: name: twitter as: kodcu ctype: name: tweets as: tweets batch: 300 # direction: me mongo: host: mongo port: 27017 elastic: host: es port: 9300 ================================================ FILE: src/test/resources/conf2 ================================================ { "misc": { "dindex": { "name": "twitter" }, "ctype": { "name": "tweets", "as": "posts" }, "direction": "em" }, "mongo": { "host": "127.0.0.1", "port": 27017 }, "elastic": { "host": "127.0.0.1", "port": 9300 } } ================================================ FILE: src/test/resources/conf3 ================================================ misc: dindex: name: twitter as: twt ctype: name: tweets as: posts batch: 500 # direction: me mongo: host: mongo_auth port: 27017 auth: user: hakan pwd: "1234" source: admin mechanism: scram-sha-1 elastic: host: es port: 9300