Full Code of ozlerhakan/mongolastic for AI

Repository: ozlerhakan/mongolastic
Branch: master
Commit: e7456115f39c
Files: 32
Total size: 62.5 KB

Directory structure:
gitextract_t9r72zye/

├── .gitignore
├── .travis.yml
├── Contributors.adoc
├── Dockerfile
├── LICENSE
├── README.adoc
├── pom.xml
└── src/
    ├── main/
    │   ├── java/
    │   │   └── com/
    │   │       └── kodcu/
    │   │           ├── config/
    │   │           │   ├── ElasticConfiguration.java
    │   │           │   ├── FileConfiguration.java
    │   │           │   ├── MongoConfiguration.java
    │   │           │   ├── YamlConfiguration.java
    │   │           │   └── structure/
    │   │           │       ├── Auth.java
    │   │           │       ├── Elastic.java
    │   │           │       ├── Misc.java
    │   │           │       ├── Mongo.java
    │   │           │       └── Namespace.java
    │   │           ├── listener/
    │   │           │   └── BulkProcessorListener.java
    │   │           ├── main/
    │   │           │   └── Mongolastic.java
    │   │           ├── provider/
    │   │           │   ├── ElasticToMongoProvider.java
    │   │           │   ├── MongoToElasticProvider.java
    │   │           │   └── Provider.java
    │   │           ├── service/
    │   │           │   ├── BulkService.java
    │   │           │   ├── ElasticBulkService.java
    │   │           │   └── MongoBulkService.java
    │   │           └── util/
    │   │               └── codecs/
    │   │                   ├── CustomDateCodec.java
    │   │                   └── CustomLongCodec.java
    │   └── resources/
    │       └── log4j2.properties
    └── test/
        ├── java/
        │   └── com/
        │       └── kodcu/
        │           └── test/
        │               ├── TestMongoToElastic.java
        │               └── TestMongolasticQueries.java
        └── resources/
            ├── conf1
            ├── conf2
            └── conf3

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
!.gitignore
*.iml
target
.idea
*.log

*.class
/.classpath
/.project
/.settings

dependency-reduced-pom.xml



================================================
FILE: .travis.yml
================================================
sudo: required
notifications:
  email: false
language: java
jdk:
- oraclejdk8
os:
- linux
services:
- docker
env:
- MONGOLASTIC_FULL="ozlerhakan/mongolastic:1.4.4"
script: mvn install
before_install:
- curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.1.1.deb && sudo dpkg -i --force-confnew elasticsearch-6.1.1.deb
- sudo service elasticsearch start
- sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6
- echo "deb http://repo.mongodb.org/apt/ubuntu precise/mongodb-org/3.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.4.list
- sudo apt-get update
- sudo apt-get install -y mongodb-org
- sudo service mongod start
- mongod --version
before_script:
- wget https://dl.dropbox.com/s/jvngnitppao8hay/tweets.zip
- unzip tweets.zip
- sleep 15
- curl http://localhost:9200/
- mongorestore -h 127.0.0.1 --port 27017 dump
- mvn clean
after_success:
- docker build -t $MONGOLASTIC_FULL .
- docker tag $MONGOLASTIC_FULL ozlerhakan/mongolastic:1.4;
- docker tag $MONGOLASTIC_FULL ozlerhakan/mongolastic:latest;
- docker images
- if [ "$TRAVIS_BRANCH" == "master" ]; then
  docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD";
  docker push $MONGOLASTIC_FULL;
  docker push ozlerhakan/mongolastic:1.4;
  docker push ozlerhakan/mongolastic:latest;
  fi
deploy:
  provider: releases
  skip_cleanup: true
  api_key:
    secure: E8iX+y1svn/88ftT5fnMIfECNUNek0T/VtrkdacuFriC/cNcopcZUuIEqmCTuMIvo+iL/h4F4wYzGLadRU6iOhnPk8oMPadcR+2Pll/mz7/I4zcj/iZL8KMJGlJxLpnTKBw3QEGnWflg/S8I6g6bOcGnPhQW0AQZ6bj7FBR59Gd7abtqAkwZVki+zxXbMzMqA3VYRaMc2UEdTU6AlcR7GeCdcDFhQ/4uqmMySGGGKLAROZ+0r9BhbaZ0ivuR7Uza9l68d0FhN2wRxDhP86RM3hbZttGQw+CBw7+3IcHoJ9MpSGgatxWd3eudWypDoocVs9FxYKS3zJcVzzLWtmB1yRdY/zOcLkr7Tkz2z+LIZbjPJFjeDUgpRJPqw8ckPOEbsdmp+/0zVBp+aDdMgpa3hfXkchTa4hXUDWJCZiuN4oByWxnBqoALUAjRdIflCeZuukbmHzZb6QWNTfRSGAZtQHO+sdJIvMFFC9v0k6T2WXvlEWDXt0cA8Gid9iBZLZfemJiBRNRP94TDcakMZzwnBH/G3l1nc7tk8AR7gX0b/biLeloSdxdI7UGV+uLvy+LgrsJp5TeLvBaylCRJ6672IvroGZwZm/GG3NsPS1vb2WoZfbjiS+paWUxreeFumJyg43Vuv0LiDND2y2P8zTtnC7HaM0k54ewt86Kx5w86Ma0=
  file: /home/travis/build/ozlerhakan/mongolastic/target/mongolastic.jar
  on:
    tags: true
    all_branches: true
    repo: ozlerhakan/mongolastic



================================================
FILE: Contributors.adoc
================================================
== Contributors

Thank you for our contributors to make this project more usable! ✨

* https://github.com/hakdogan[Hüseyin Hakdoğan]
* https://github.com/winder[Will Winder]
* https://github.com/wareninja[Yılmaz Güleryüz]

================================================
FILE: Dockerfile
================================================
FROM openjdk:8-jdk-alpine
MAINTAINER Hakan Ozler <ozler.hakan@gmail.com>
ADD target/mongolastic.jar .
ENTRYPOINT ["java","-jar","mongolastic.jar","-f"]

================================================
FILE: LICENSE
================================================
The MIT License (MIT)

Copyright (c) 2015-2016 Kodcu.com

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.


================================================
FILE: README.adoc
================================================
= Mongolastic
:version: v1.4.3

image:https://travis-ci.org/ozlerhakan/mongolastic.svg?branch=master["Build Status", link="https://travis-ci.org/ozlerhakan/mongolastic"]
image:https://api.codacy.com/project/badge/Grade/8d768c2fc37246019115e4d090a33b98["Codacy code quality", link="https://www.codacy.com/app/ozlerhakan/mongolastic?utm_source=github.com&utm_medium=referral&utm_content=ozlerhakan/mongolastic&utm_campaign=Badge_Grade"]
image:https://img.shields.io/docker/pulls/ozlerhakan/mongolastic.svg["Docker Pulls,link="https://hub.docker.com/r/ozlerhakan/mongolastic"]
image:https://img.shields.io/github/release/ozlerhakan/mongolastic.svg[]
image:https://img.shields.io/badge/mongo.java.driver-3.4.2-brightgreen.svg[] image:https://img.shields.io/badge/elastic.java.driver-6.2.4-brightgreen.svg[]
image:https://img.shields.io/badge/license-MIT-blue.svg[]

Mongolastic enables you to migrate your datasets from a mongod node to an elasticsearch node and vice versa. Since mongo and elastic servers can run with different characteristics, the tool provides several optional and required features to ably connect them. Mongolastic works with a yaml or json configuration file to begin a migration process. It reads your demand on the file and start syncing data in the specified direction.

== How it works

First, you can either pull the corresponding image of the app from https://hub.docker.com/r/ozlerhakan/mongolastic/[Docker Hub]

Supported tags and respective Dockerfile links:

*  `_1.4_`, `_1.4.4_`, `_latest_` https://github.com/ozlerhakan/mongolastic/blob/master/Dockerfile[_(master/Dockerfile)_]
*  `_1.4.3_` https://github.com/ozlerhakan/mongolastic/blob/0dacd80cbdf7b5b7b282bf6dd89ede8558021577/Dockerfile[_(1.4.2/Dockerfile)_]


or download the latest https://github.com/ozlerhakan/mongolastic/releases/download/{version}/mongolastic.jar[mongolastic.jar] file.

Second, create a yaml or json file which must contain the following structure:

[source,yaml]
----
misc:
    dindex:
        name: <string>      <1>
        as: <string>        <2>
    ctype:
        name: <string>      <3>
        as: <string>        <4>
    direction: (em | me)    <5>
    batch: <number>         <6>
    dropDataset: <bool>     <7>
mongo:
    host: <ip-address>      <8>
    port: <number>          <9>
    query: "mongo-query"    <10>
    project: "projection"   <11>
    auth:                   <12>
        user: <string>
        pwd: "password"
        source: <db-name>
        mechanism: ( plain | scram-sha-1 | x509 | gssapi | cr )
elastic:
    host: <ip-address>     <13>
    port: <number>         <14>
    dateFormat: "<format>" <15>
    longToString: <bool>   <16>
    clusterName: <string>  <17>
    auth:                  <18>
        user: <string>
        pwd: "password"
----
<1>  the _database/index name_ to connect to.
<2>  another _database/index name_ in which documents will be located in the target service (*Optional*)
<3>  the _collection/type name_ to export.
<4>  another _collection/type name_ in which indexed/collected documents will reside in the target service (*Optional*)
<5>  direction of the data transfer. the default direction is me (that is, mongo to elasticsearch). You can skip this option if your data move from mongo to es.
<6>  Override the default batch size which is normally 200. (*Optional*)
<7>  configures whether or not the target table should be dropped prior to loading data. Default value is true (*Optional*)
<8>  the name of the host machine where the `mongod` is running.
<9>  the port where the `mongod` instance is listening.
<10>  data will be transferred based on a json mongodb query (*Optional*)
<11> with 1.4.1, you can manipulate documents that will be migrated from mongo to es based on the https://docs.mongodb.com/manual/reference/operator/aggregation/project/[`$project`] operator (*Optional*)
<12> as of v1.3.5, you can access an auth mongodb by giving auth configuration. (*Optional*)
<13> the name of the host machine where the `elastic node` is running.
<14> the *transport* port where the transport module will communicate with the running elastic node. E.g. *9300* for node-to-node communication.
<15> a custom formatter for Date fields rather than the default DateCodec (*Optional*)
<16> serialize long value as a string for backwards compatibility with other tools (*Optional*)
<17> connect to a spesific elastic cluster (*Optional*)
<18> as of v1.3.9, you can access an auth elastic search by giving auth configuration. (*Optional*)

---

Alternatively, a JSON file can be specified as a mongolastic configuration file including the same YAML file structure above.

[source,json]
----
{
	"misc": {
		"dindex": {
			"name": "twitter",
			"as": "media"
		},
		"ctype": {
			"name": "tweets",
			"as": "posts"
		},
		"direction": "me",
		"batch": 400,
		"dropDataset": true
	},
	"mongo": {
		"host": "127.0.0.1",
		"port": 27017,
		"query": "{ lang: 'en' }",
		"project": "{ user:1, name:'$user.name', location: { $substr: [ '$user.location', 10, 15 ] }}",
		"auth": {
			"user": "joe",
			"pwd": "1234",
			"source": "twitter",
			"mechanism": "scram-sha-1"
		}
	},
	"elastic": {
		"host": "127.0.0.1",
		"port": 9300,
		"dateFormat": "yyyy-MM-dd",
		"longToString": true,
		"auth": {
			"user": "joe",
			"pwd": "4321"
		}
	}
}
----

== Example #1

The following files have the same configuration details:

.yaml file
[source,yaml]
----
misc:
    dindex:
        name: twitter
        as: kodcu
    ctype:
        name: tweets
        as: posts
mongo:
    host: localhost
    port: 27017
    query: "{ 'user.name' : 'kodcu.com'}"
elastic:
    host: localhost
    port: 9300
----

.json file
[source,json]
----
{
	"misc": {
		"dindex": {
			"name": "twitter",
			"as": "kodcu"
		},
		"ctype": {
			"name": "tweets",
			"as": "posts"
		}
	},
	"mongo": {
		"host": "localhost",
		"port": 27017,
		"query": "{ 'user.name' : 'kodcu.com'}"
	},
	"elastic": {
		"host": "localhost",
		"port": 9300
	}
}
----

the config says that the transfer direction is from mongodb to elasticsearch, mongolastic first looks at the _tweets_ collection, where the _user name_ is _kodcu.com_, of the _twitter_ database located on a mongod server running on default host interface and port number. If It finds the corresponding data, It will start copying those into an elasticsearch environment running on default host and transport number. After all, you should see a type called _"posts"_ in an index called _"kodcu"_ in the current elastic node. Why the index and type are different is because "dindex.as" and "ctype.as" options were set, these indicates that your data being transferred exist in _posts_ type of the _kodcu_ index.

After downloading the jar or pulling the image and providing a conf file, you can either run the tool as:

    $ java -jar mongolastic.jar -f config.file

__or__

    $ docker run --rm -v $(PWD)/config.file:/config.file --net host ozlerhakan/mongolastic:<tag> config.file

== Example #2

Using the project field, you are able to manipulate documents when migrating them from mongodb to elasticsearch. For more examples about the `$project` operator of the aggregation pipeline, take a look at its https://docs.mongodb.com/manual/reference/operator/aggregation/project/[documentation].

[source,yaml]
----
misc:
    dindex:
        name: twitter
    ctype:
        name: tweets
mongo:
    host: 192.168.10.151
    port: 27017
    project: "{ user: 1, name: '$user.name', location: { $substr: [ '$user.location', 10, 15 ] }}" <1>
elastic:
    host: 192.168.10.152
    port: 9300
----
<1> the migrated documents will include the user field and contain new fields `name` and `location`.

NOTE: Every attempt of running the tool drops the mentioned db/index in the target environment unless the dropDataset parameter is configured otherwise.

== License

Mongolastic is released under http://showalicense.com/?hide_explanations=false&year=2015&fullname=Kodcu.com#license-mit[MIT].


================================================
FILE: pom.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xmlns="http://maven.apache.org/POM/4.0.0"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.kodcu.mongolastic</groupId>
    <artifactId>mongolastic</artifactId>
    <version>1.4.4</version>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <java.version>1.8</java.version>
        <elastic.version>6.2.4</elastic.version>
        <mongo.version>3.4.2</mongo.version>
        <logger.version>2.8</logger.version>
    </properties>

    <inceptionYear>2015</inceptionYear>

    <name>mongolastic</name>
    <description>
        A tool that migrates data from mongodb to elasticsearch and vice versa.
    </description>
    <url>http://github.com/ozlerhakan/mongolastic</url>

    <organization>
        <name>Kodcu.com</name>
    </organization>

    <developers>
        <developer>
            <id>ozlerhakan</id>
            <name>Hakan Ozler</name>
            <email>ozler.hakan@gmail.com</email>
            <url>http://github.com/ozlerhakan</url>
            <roles>
                <role>developer</role>
            </roles>
        </developer>
        <developer>
            <id>hakdogan</id>
            <name>Hüseyin Akdoğan</name>
            <email>huseyin.akdogan@kodcu.com</email>
            <url>http://github.com/hakdogan</url>
            <roles>
                <role>software evangelist - developer</role>
            </roles>
        </developer>
    </developers>

    <prerequisites>
        <maven>3.0</maven>
    </prerequisites>

    <issueManagement>
        <system>github.com</system>
        <url>https://github.com/ozlerhakan/mongolastic/issues</url>
    </issueManagement>

    <repositories>
        <!-- add the elasticsearch repo -->
        <repository>
            <id>elasticsearch-releases</id>
            <url>https://artifacts.elastic.co/maven</url>
            <releases>
                <enabled>true</enabled>
            </releases>
            <snapshots>
                <enabled>false</enabled>
            </snapshots>
        </repository>
    </repositories>

    <dependencies>
        <dependency>
            <groupId>org.mongodb</groupId>
            <artifactId>mongodb-driver</artifactId>
            <version>${mongo.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-log4j12</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>log4j</groupId>
                    <artifactId>log4j</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>transport</artifactId>
            <version>${elastic.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-log4j12</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>log4j</groupId>
                    <artifactId>log4j</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-slf4j-impl</artifactId>
            <version>${logger.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-api</artifactId>
            <version>${logger.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>${logger.version}</version>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.8.6</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
            <scope>test</scope>
        </dependency>
    </dependencies>


    <build>
        <finalName>${project.artifactId}</finalName>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.0.0</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <transformers>
                                <transformer
                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                    <mainClass>com.kodcu.main.Mongolastic</mainClass>
                                    <manifestEntries>
                                        <Change></Change>
                                        <Build-Date></Build-Date>
                                    </manifestEntries>
                                </transformer>
                                <transformer
                                        implementation="org.apache.maven.plugins.shade.resource.PluginXmlResourceTransformer"/>
                                <transformer
                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                            </transformers>
                            <filters>
                                <filter>
                                    <artifact>*:*</artifact>
                                    <excludes>
                                        <exclude>META-INF/*.SF</exclude>
                                        <exclude>META-INF/*.DSA</exclude>
                                        <exclude>META-INF/*.RSA</exclude>
                                    </excludes>
                                </filter>
                            </filters>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.6.1</version>
                <configuration>
                    <source>${java.version}</source>
                    <target>${java.version}</target>
                </configuration>
            </plugin>
        </plugins>
    </build>
</project>


================================================
FILE: src/main/java/com/kodcu/config/ElasticConfiguration.java
================================================
package com.kodcu.config;

import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Objects;
import java.util.Optional;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.Settings.Builder;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;

/**
 * Created by hakdogan on 21/05/15.
 */
public class ElasticConfiguration {

    private final Logger logger = LogManager.getLogger(ElasticConfiguration.class);
    private final YamlConfiguration config;
    private Client client;

    public ElasticConfiguration(final YamlConfiguration config) {
        this.config = config;
        this.prepareClient();
    }

    private void prepareClient() {

        Builder settingsBuilder = applySettings();
        try {
            TransportAddress ista = new TransportAddress(InetAddress.getByName(config.getElastic().getHost()), config.getElastic().getPort());
            client = new PreBuiltTransportClient(settingsBuilder.build())
                    .addTransportAddress(ista);

        } catch (UnknownHostException ex) {
            logger.error(ex.getMessage(), ex);
            System.exit(-1);
        }
    }

    private Builder applySettings() {
        Builder settingsBuilder = Settings.builder();

        settingsBuilder.put("client.transport.ping_timeout", "15s");
        settingsBuilder.put("client.transport.nodes_sampler_interval", "5s");
        // YG: to ensure reliable connection & resolve NoNodeAvailableException
        settingsBuilder.put("client.transport.sniff", true);
        settingsBuilder.put("network.bind_host", 0);

        // YG: for supporting ES Auth with ES Shield
        Optional.ofNullable(config.getElastic().getAuth())
                .ifPresent(auth -> settingsBuilder.put("xpack.security.user", String.join(":", auth.getUser(), auth.getPwd())));

        if (Objects.nonNull(config.getElastic().getClusterName())) {
            settingsBuilder.put("cluster.name", config.getElastic().getClusterName());
        } else {
            settingsBuilder.put("client.transport.ignore_cluster_name", true);
        }
        return settingsBuilder;
    }

    public void closeNode() {
        client.close();
    }

    public Client getClient() {
        return client;
    }

}


================================================
FILE: src/main/java/com/kodcu/config/FileConfiguration.java
================================================
package com.kodcu.config;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Objects;

/**
 * Created by Hakan on 5/19/2015.
 */
public class FileConfiguration {

    private final Logger logger = LoggerFactory.getLogger(FileConfiguration.class);
    private final String parameter;

    public FileConfiguration(String parameter) {
        this.parameter = parameter;
    }

    public YamlConfiguration getFileContent() {
        YamlConfiguration config = null;
        File ymlFile = new File(parameter);
        try {
            Yaml yaml = new Yaml();
            if (ymlFile.isFile()) {
                FileInputStream configFile = new FileInputStream(ymlFile);
                config = yaml.loadAs(configFile, YamlConfiguration.class);
            } else {
                // we expect that this is just a string including yaml format
                config = yaml.loadAs(parameter, YamlConfiguration.class);
            }
        } catch (Exception e) {
            try {
                ObjectMapper mapper = new ObjectMapper();
                config = mapper.readValue(ymlFile, YamlConfiguration.class);
            } catch (IOException ex) {
                logger.error(e.getMessage(), e);
                System.exit(0);
            }
        }

        logger.info(System.lineSeparator() + "Config Output:" + System.lineSeparator() + config.toString() + System.lineSeparator());
        config = this.controlAsSettings(config);
        return config;
    }

    private YamlConfiguration controlAsSettings(YamlConfiguration config) {
        String dIndexAs = config.getMisc().getDindex().getAs();
        String cTypeAs = config.getMisc().getCtype().getAs();
        if (Objects.isNull(dIndexAs))
            config.getMisc().getDindex().setAs(config.getMisc().getDindex().getName());
        if (Objects.isNull(cTypeAs))
            config.getMisc().getCtype().setAs(config.getMisc().getCtype().getName());
        if (config.getMisc().getBatch() < 200)
            config.getMisc().setBatch(200);

        return config;
    }

}


================================================
FILE: src/main/java/com/kodcu/config/MongoConfiguration.java
================================================
package com.kodcu.config;

import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.MongoCredential;
import com.mongodb.ReadPreference;
import com.mongodb.ServerAddress;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import java.util.Objects;

import static com.mongodb.assertions.Assertions.notNull;

/**
 * Created by Hakan on 5/19/2015.
 */
public class MongoConfiguration {

    private final Logger logger = LoggerFactory.getLogger(MongoConfiguration.class);
    private final YamlConfiguration config;
    private MongoClient client;

    public MongoConfiguration(final YamlConfiguration config) {
        this.config = config;
        this.prepareClient();
    }

    private void prepareClient() {
        try {
            ServerAddress address = new ServerAddress(config.getMongo().getHost(), config.getMongo().getPort());
            MongoClientOptions options = MongoClientOptions.builder()
                    .serverSelectionTimeout(5000)
                    .socketKeepAlive(false)
                    .readPreference(ReadPreference.primaryPreferred())
                    .sslInvalidHostNameAllowed(true)
                    .build();

             client = connectToClient(address, options);
        } catch (Exception ex) {
            logger.error(ex.getMessage(), ex);
            System.exit(-1);
        }
    }

    private MongoClient connectToClient(ServerAddress address, MongoClientOptions options) {
        if (Objects.nonNull(config.getMongo().getAuth())) {

            String user = notNull("auth.name", config.getMongo().getAuth().getUser());
            String database = config.getMongo().getAuth().getSource();
            char[] pwd = config.getMongo().getAuth().getPwd().toCharArray();
            String mechanism = config.getMongo().getAuth().getMechanism();

            MongoCredential credential = findMongoCredential(user, database, pwd, mechanism);
            return new MongoClient(Arrays.asList(address), Arrays.asList(credential), options);

        } else {
            return new MongoClient(Arrays.asList(address), options);
        }
    }

    private MongoCredential findMongoCredential(String user, String database, char[] pwd, String mechanism) {
        MongoCredential credential = null;
        switch (mechanism) {
            case "scram-sha-1":
                credential = MongoCredential.createScramSha1Credential(user, database, pwd);
                break;
            case "x509":
                credential = MongoCredential.createMongoX509Credential(user);
                break;
            case "cr":
                credential = MongoCredential.createMongoCRCredential(user, database, pwd);
                break;
            case "plain":
                credential = MongoCredential.createPlainCredential(user, database, pwd);
                break;
            case "gssapi":
                credential = MongoCredential.createGSSAPICredential(user);
                break;
            default:
                credential = MongoCredential.createCredential(user, database, pwd);
                break;
        }
        return credential;
    }

    public MongoCollection<Document> getMongoCollection() {
        MongoCollection<Document> collection = null;
        try {
            MongoDatabase database = client.getDatabase(config.getMisc().getDindex().getName());
            collection = database.getCollection(config.getMisc().getCtype().getName());
        } catch (Exception ex) {
            logger.error(ex.getMessage(), ex);
        }
        return collection;
    }

    public void closeConnection() {
        if (Objects.nonNull(client))
            client.close();
    }

    public MongoClient getClient() {
        return client;
    }
}


================================================
FILE: src/main/java/com/kodcu/config/YamlConfiguration.java
================================================
package com.kodcu.config;

import com.kodcu.config.structure.Elastic;
import com.kodcu.config.structure.Misc;
import com.kodcu.config.structure.Mongo;

/**
 * Created by Hakan on 5/19/2015.
 */
public class YamlConfiguration {

    private Misc misc;
    private Mongo mongo;
    private Elastic elastic;

    public Elastic getElastic() {
        return elastic;
    }

    public void setElastic(Elastic elastic) {
        this.elastic = elastic;
    }

    public Misc getMisc() {
        return misc;
    }

    public void setMisc(Misc misc) {
        this.misc = misc;
    }

    public Mongo getMongo() {
        return mongo;
    }

    public void setMongo(Mongo mongo) {
        this.mongo = mongo;
    }

    @Override
    public String toString() {
        return "{" +
                "elastic=" + elastic +
                ", misc=" + misc +
                ", mongo=" + mongo +
                '}';
    }
}

================================================
FILE: src/main/java/com/kodcu/config/structure/Auth.java
================================================
package com.kodcu.config.structure;

/**
 * Created by Hakan on 5/14/2016.
 */
public class Auth {

    private String user;
    private String pwd;
    private String source;
    private String mechanism;

    public String getUser() {
        return user;
    }

    public void setUser(String user) {
        this.user = user;
    }

    public String getPwd() {
        return pwd;
    }

    public void setPwd(String pwd) {
        this.pwd = pwd;
    }

    public String getSource() {
        return source;
    }

    public void setSource(String source) {
        this.source = source;
    }

    public String getMechanism() {
        return mechanism;
    }

    public void setMechanism(String mechanism) {
        this.mechanism = mechanism;
    }

    @Override
    public String toString() {
        return "Auth [user=" + user + ", pwd=" + pwd + ", source=" + source + ", mechanism=" + mechanism + "]";
    }

}


================================================
FILE: src/main/java/com/kodcu/config/structure/Elastic.java
================================================
package com.kodcu.config.structure;

/**
 * Created by Hakan on 1/16/2016.
 */
public class Elastic {

    private String host;
    private int port;
    private String dateFormat;
    private Boolean longToString = false;
    private String clusterName; // optional
    private Auth auth; // optional

    public Auth getAuth() {
        return auth;
    }

    public void setAuth(Auth auth) {
        this.auth = auth;
    }

    public String getClusterName() {
        return clusterName;
    }

    public void setClusterName(String clusterName) {
        this.clusterName = clusterName;
    }

    public String getHost() {
        return host;
    }

    public void setHost(String host) {
        this.host = host;
    }

    public int getPort() {
        return port;
    }

    public void setPort(int port) {
        this.port = port;
    }

    public String getDateFormat() {
        return dateFormat;
    }

    public void setDateFormat(String dateFormat) {
        this.dateFormat = dateFormat;
    }

    public Boolean getLongToString() {
        return this.longToString;
    }

    public void setLongToString(Boolean longToString) {
        this.longToString = longToString;
    }

    @Override
    public String toString() {
        return "Elastic{" +
                "host='" + host + '\'' +
                ", port=" + port +
                ", clusterName=" + clusterName +
                ", dateFormat=" + dateFormat +
                ", longToString=" + longToString +
                ", auth=" + auth +
                '}';
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        if (o == null || getClass() != o.getClass()) return false;

        Elastic elastic = (Elastic) o;

        if (port != elastic.port) return false;
        return host != null ? host.equals(elastic.host) : elastic.host == null;

    }

    @Override
    public int hashCode() {
        int result = host != null ? host.hashCode() : 0;
        result = 31 * result + port;
        return result;
    }
}


================================================
FILE: src/main/java/com/kodcu/config/structure/Misc.java
================================================
package com.kodcu.config.structure;

/**
 * Created by Hakan on 1/16/2016.
 */
public class Misc {

    private String direction = "me";
    private Namespace dindex;
    private Namespace ctype;
    private Boolean dropDataset = true;
    private int batch = 200;

    public int getBatch() {
        return batch;
    }

    public void setBatch(int batch) {
        this.batch = batch;
    }

    public Namespace getCtype() {
        return ctype;
    }

    public void setCtype(Namespace ctype) {
        this.ctype = ctype;
    }

    public String getDirection() {
        return direction;
    }

    public void setDirection(String direction) {
        this.direction = direction;
    }

    public Namespace getDindex() {
        return dindex;
    }

    public void setDindex(Namespace dindex) {
        this.dindex = dindex;
    }

    public Boolean getDropDataset() {
        return this.dropDataset;
    }

    public void setDropDataset(Boolean dropDataset) {
        this.dropDataset = dropDataset;
    }

    @Override
    public String toString() {
        return "Misc{" +
                "batch=" + batch +
                ", direction='" + direction + '\'' +
                ", dindex=" + dindex +
                ", ctype=" + ctype +
                ", dropDataset=" + dropDataset +
                '}';
    }
}


================================================
FILE: src/main/java/com/kodcu/config/structure/Mongo.java
================================================
package com.kodcu.config.structure;

/**
 * Created by Hakan on 1/16/2016.
 */
public class Mongo {

    private String host;
    private int port;
    private String query = "{}";
    private String project;
    private Auth auth;

    @Override
    public String toString() {
        return "Mongo{" +
                "host='" + host + '\'' +
                ", port=" + port +
                ", query='" + query + '\'' +
                ", project='" + project + '\'' +
                ", auth=" + auth +
                '}';
    }

    public String getProject() {
        return project;
    }

    public void setProject(String project) {
        this.project = project;
    }

    public Auth getAuth() {
        return auth;
    }

    public void setAuth(Auth auth) {
        this.auth = auth;
    }

    public String getQuery() {
        return query;
    }

    public void setQuery(String query) {
        this.query = query;
    }

    public int getPort() {
        return port;
    }

    public void setPort(int port) {
        this.port = port;
    }

    public String getHost() {
        return host;
    }

    public void setHost(String host) {
        this.host = host;
    }
}


================================================
FILE: src/main/java/com/kodcu/config/structure/Namespace.java
================================================
package com.kodcu.config.structure;

/**
 * Created by Hakan on 1/16/2016.
 */
public class Namespace {
    private String name;
    private String as;

    public String getAs() {
        return as;
    }

    public void setAs(String as) {
        this.as = as;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    @Override
    public String toString() {
        return "Namespace{" +
                "as='" + as + '\'' +
                ", name='" + name + '\'' +
                '}';
    }
}


================================================
FILE: src/main/java/com/kodcu/listener/BulkProcessorListener.java
================================================
package com.kodcu.listener;

import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Created by Hakan on 5/21/2015.
 */
public class BulkProcessorListener implements BulkProcessor.Listener {

    private final Logger logger = LoggerFactory.getLogger(BulkProcessorListener.class);

    @Override
    public void beforeBulk(long executionId, BulkRequest request) {
    }

    @Override
    public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
        if (response.hasFailures()) {
            logger.error(response.buildFailureMessage());
        } else {
            logger.info(String.format("Data transfer successfully terminated.(%d)", response.getItems().length));
        }

    }

    @Override
    public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
        logger.error("Transfer failed.");
        logger.error(failure.getMessage(), failure.fillInStackTrace());
    }
}


================================================
FILE: src/main/java/com/kodcu/main/Mongolastic.java
================================================
package com.kodcu.main;

import com.kodcu.config.ElasticConfiguration;
import com.kodcu.config.FileConfiguration;
import com.kodcu.config.MongoConfiguration;
import com.kodcu.config.YamlConfiguration;
import com.kodcu.provider.ElasticToMongoProvider;
import com.kodcu.provider.MongoToElasticProvider;
import com.kodcu.provider.Provider;
import com.kodcu.service.BulkService;
import com.kodcu.service.ElasticBulkService;
import com.kodcu.service.MongoBulkService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Optional;

/**
 * Created by Hakan on 5/19/2015.
 */
public class Mongolastic {

    private static final Logger logger = LoggerFactory.getLogger(Mongolastic.class);
    private final String parameter;

    public Mongolastic(String parameter) {
        this.parameter = parameter;
    }

    public static void main(String[] args) throws Exception {
        configAssertion(args);

        String parameter = args[1];
        Mongolastic app = new Mongolastic(parameter);
        app.start();
    }

    private static void configAssertion(String[] args) {
        if (args.length == 0) {
            logger.error("Incorrect syntax. Should be mongolastic.jar -f /path/file");
            System.exit(0);
        }
        if (!args[0].equals("-f")) {
            logger.error("Please specify the -f parameter with a correct yaml or json file");
            System.exit(0);
        }
        if (args.length != 2) {
            logger.error("Incorrect syntax. Pass max 2 parameters");
            System.exit(0);
        }
    }

    public void start() {
        FileConfiguration fConfig = new FileConfiguration(parameter);
        Optional<YamlConfiguration> yamlConfig = Optional.ofNullable(fConfig.getFileContent());
        long begin = System.currentTimeMillis();
        try {
            yamlConfig.ifPresent(this::proceedService);
        } finally {
            logger.info("Load duration: " + (System.currentTimeMillis() - begin) + "ms");
        }
    }

    public void proceedService(YamlConfiguration config) {
        ElasticConfiguration elastic = new ElasticConfiguration(config);
        MongoConfiguration mongo = new MongoConfiguration(config);
        BulkService bulkService = this.initializeBulkService(config, mongo, elastic);
        Provider provider = this.initializeProvider(config, mongo, elastic);
        provider.transfer(bulkService, config, () -> {
            bulkService.close();
            mongo.closeConnection();
            elastic.closeNode();
        });
    }

    private Provider initializeProvider(YamlConfiguration config, MongoConfiguration mongo, ElasticConfiguration elastic) {
        if (config.getMisc().getDirection().equals("em")) {
            return new ElasticToMongoProvider(elastic, config);
        }
        return new MongoToElasticProvider(mongo.getMongoCollection(), config);
    }

    private BulkService initializeBulkService(YamlConfiguration config, MongoConfiguration mongo, ElasticConfiguration elastic) {
        if (config.getMisc().getDirection().equals("em")) {
            return new MongoBulkService(mongo.getClient(), config);
        }
        return new ElasticBulkService(config, elastic);
    }
}


================================================
FILE: src/main/java/com/kodcu/provider/ElasticToMongoProvider.java
================================================
package com.kodcu.provider;

import com.kodcu.config.ElasticConfiguration;
import com.kodcu.config.YamlConfiguration;
import org.bson.Document;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.common.unit.TimeValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;

/**
 * Created by Hakan on 6/29/2015.
 */
public class ElasticToMongoProvider implements Provider {

    private final Logger logger = LoggerFactory.getLogger(ElasticToMongoProvider.class);
    private final ElasticConfiguration elastic;
    private final YamlConfiguration config;
    private SearchResponse response;

    public ElasticToMongoProvider(final ElasticConfiguration elastic, final YamlConfiguration config) {
        this.elastic = elastic;
        this.config = config;
    }

    @Override
    public long getCount() {
        long count = 0;
        IndicesAdminClient admin = elastic.getClient().admin().indices();
        IndicesExistsRequestBuilder builder = admin.prepareExists(config.getMisc().getDindex().getName());
        if (builder.execute().actionGet().isExists()) {
            SearchResponse countResponse = elastic.getClient().prepareSearch(config.getMisc().getDindex().getName())
                    .setTypes(config.getMisc().getCtype().getName())
                    .setSearchType(SearchType.QUERY_THEN_FETCH)
                    .setSize(0)
                    .execute().actionGet();
            count = countResponse.getHits().getTotalHits();
        } else {
            logger.info("Index/Type does not exist or does not contain the record");
            System.exit(-1);
        }

        logger.info("Elastic Index/Type count: " + count);
        return count;
    }

    @Override
    public List buildJSONContent(int skip, int limit) {

        if (Objects.isNull(response)) {
            response = elastic.getClient().prepareSearch(config.getMisc().getDindex().getName())
                    .setTypes(config.getMisc().getCtype().getName())
                    .setSearchType(SearchType.QUERY_THEN_FETCH)
                    .setScroll(new TimeValue(60000))
                    .setSize(limit)
                    .execute().actionGet();
        }
        else {
            response = elastic.getClient()
                    .prepareSearchScroll(response.getScrollId())
                    .setScroll(new TimeValue(60000))
                    .execute().actionGet();
        }

        return Arrays.stream(response.getHits().getHits())
                .map(hit -> new Document(hit.getSourceAsMap()))
                .collect(Collectors.toList());
    }
}


================================================
FILE: src/main/java/com/kodcu/provider/MongoToElasticProvider.java
================================================
package com.kodcu.provider;

import com.kodcu.config.YamlConfiguration;
import com.mongodb.client.AggregateIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.List;
import java.util.Optional;

import static com.mongodb.client.model.Aggregates.*;

/**
 * Created by Hakan on 5/18/2015.
 */
public class MongoToElasticProvider implements Provider {

    private final Logger logger = LoggerFactory.getLogger(MongoToElasticProvider.class);
    private final MongoCollection<Document> collection;
    private final YamlConfiguration config;
    private MongoCursor<Document> cursor;
    private long cursorId = 0;

    public MongoToElasticProvider(final MongoCollection<Document> collection, final YamlConfiguration config) {
        this.collection = collection;
        this.config = config;
    }

    @Override
    public long getCount() {
        long count = collection.count(Document.parse(config.getMongo().getQuery()));
        logger.info("Mongo collection count: " + count);
        if (count == 0) {
            logger.error("Database/Collection does not exist or does not contain the record");
            System.exit(-1);
        }
        return count;
    }

    @Override
    public List buildJSONContent(int skip, int limit) {
        ArrayList<Document> result = new ArrayList<>(limit);
        result.ensureCapacity(limit);

        MongoCursor<Document> cursor = getCursor(skip);
        while (cursor.hasNext() && result.size() < limit) {
            result.add(cursor.next());
        }
        return result;
    }

    /**
     * Get the MongoDB cursor.
     */
    private MongoCursor<Document> getCursor(int skip) {
        if (cursor == null && cursorId == 0) {
            Document query = Document.parse(config.getMongo().getQuery());
            List<Bson> pipes = new ArrayList<>(3);
            pipes.add(match(query));
            pipes.add(skip(skip));

            Optional.ofNullable(config.getMongo().getProject()).ifPresent(p -> pipes.add(project(Document.parse(p))));

            AggregateIterable<Document> aggregate = collection.aggregate(pipes)
                    .allowDiskUse(true)
                    .useCursor(true);

            cursor = aggregate.iterator();

            // TODO: Persist cursor ID somewhere to allow restarts.
            Optional.ofNullable(cursor.getServerCursor()).ifPresent(serverCursor -> cursorId = serverCursor.getId());
        } else if (cursor == null && cursorId != 0) {
            // TODO: Lookup cursor ID for resume.
            // Open existing cursor in case of restart??
        }

        return cursor;
    }
}


================================================
FILE: src/main/java/com/kodcu/provider/Provider.java
================================================
package com.kodcu.provider;


import com.kodcu.config.YamlConfiguration;
import com.kodcu.service.BulkService;
import org.bson.Document;

import java.util.List;

/**
 * Created by Hakan on 6/30/2015.
 */
public interface Provider {

    default void transfer(final BulkService bulkService, final YamlConfiguration config, final Runnable closeConnections) {
        long count = this.getCount();
        final int limit = config.getMisc().getBatch();
        int skip = 0;

        if (count != 0 && config.getMisc().getDropDataset())
            bulkService.dropDataSet();

        while (count >= limit) {
            List content = this.buildJSONContent(skip, limit);
            bulkService.proceed(content);
            count -= limit;
            skip += limit;
        }

        if (count > 0) {
            List content = this.buildJSONContent(skip, (int) count);
            bulkService.proceed(content);
        }

        closeConnections.run();
    }

    long getCount();

    List<Document> buildJSONContent(int skip, int limit);
}


================================================
FILE: src/main/java/com/kodcu/service/BulkService.java
================================================
package com.kodcu.service;

import java.util.List;

/**
 * Created by Hakan on 6/30/2015.
 */
public interface BulkService {

    void proceed(List content);

    void dropDataSet();

    void close();
}


================================================
FILE: src/main/java/com/kodcu/service/ElasticBulkService.java
================================================
package com.kodcu.service;

import com.kodcu.config.ElasticConfiguration;
import com.kodcu.config.YamlConfiguration;
import com.kodcu.listener.BulkProcessorListener;
import com.kodcu.util.codecs.CustomDateCodec;
import com.kodcu.util.codecs.CustomLongCodec;
import com.mongodb.MongoClient;
import org.bson.Document;
import org.bson.codecs.BsonTypeClassMap;
import org.bson.codecs.Codec;
import org.bson.codecs.DocumentCodec;
import org.bson.codecs.Encoder;
import org.bson.codecs.configuration.CodecRegistries;
import org.bson.codecs.configuration.CodecRegistry;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder;
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

/**
 * Created by Hakan on 5/21/2015.
 */
public class ElasticBulkService implements BulkService {

    private final Logger logger = LoggerFactory.getLogger(ElasticBulkService.class);
    private final YamlConfiguration config;
    private final ElasticConfiguration client;
    private final BulkProcessor bulkProcessor;
    private final Encoder<Document> encoder;

    public ElasticBulkService(final YamlConfiguration config, final ElasticConfiguration client) {
        this.config = config;
        this.client = client;

        this.bulkProcessor = BulkProcessor.builder(client.getClient(), new BulkProcessorListener())
                .setBulkActions(config.getMisc().getBatch())
                .setFlushInterval(TimeValue.timeValueSeconds(5))
                .setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB))
                .build();

        encoder = getEncoder();
    }

    @Override
    public void proceed(List content) {
        try {
            logger.info("Transferring data began to elasticsearch.");
            final String indexName = config.getMisc().getDindex().getAs();
            final String typeName = config.getMisc().getCtype().getAs();
            for (Object o : content) {
                Document doc = (Document) o;
                Object id = doc.get("_id");
                IndexRequest indexRequest = new IndexRequest(indexName, typeName, String.valueOf(id));
                doc.remove("_id");
                indexRequest.source(doc.toJson(encoder), XContentType.JSON);
                bulkProcessor.add(indexRequest);
            }
        } catch (Exception ex) {
            logger.debug(ex.getMessage(), ex);
        }
    }

    @Override
    public void close() {
        try {
            bulkProcessor.awaitClose(10, TimeUnit.MINUTES);
        } catch (InterruptedException ex) {
            logger.error(ex.getMessage(), ex);
        }
    }

    @Override
    public void dropDataSet() {
        final String indexName = config.getMisc().getDindex().getAs();
        IndicesAdminClient admin = client.getClient().admin().indices();
        IndicesExistsRequestBuilder builder = admin.prepareExists(indexName);
        if (builder.execute().actionGet().isExists()) {
            DeleteIndexResponse delete = admin.delete(new DeleteIndexRequest(indexName)).actionGet();
            if (delete.isAcknowledged())
                logger.info(String.format("The current index %s was deleted.", indexName));
            else
                logger.info(String.format("The current index %s was not deleted.", indexName));
        }
    }

    /**
     * Customizations for the document.toJson output.
     * <p>
     * http://mongodb.github.io/mongo-java-driver/3.0/bson/codecs/
     *
     * @return the toJson encoder.
     */
    private Encoder<Document> getEncoder() {
        ArrayList<Codec<?>> codecs = new ArrayList<>();

        if (config.getElastic().getDateFormat() != null) {
            // Replace default DateCodec class to use the custom date formatter.
            codecs.add(new CustomDateCodec(config.getElastic().getDateFormat()));
        }

        if (config.getElastic().getLongToString()) {
            // Replace default LongCodec class
            codecs.add(new CustomLongCodec());
        }

        if (codecs.size() > 0) {
            BsonTypeClassMap bsonTypeClassMap = new BsonTypeClassMap();

            CodecRegistry codecRegistry = CodecRegistries.fromRegistries(
                    CodecRegistries.fromCodecs(codecs),
                    MongoClient.getDefaultCodecRegistry());

            return new DocumentCodec(codecRegistry, bsonTypeClassMap);
        } else {
            return new DocumentCodec();
        }
    }
}


================================================
FILE: src/main/java/com/kodcu/service/MongoBulkService.java
================================================
package com.kodcu.service;

import com.kodcu.config.YamlConfiguration;
import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;

/**
 * Created by Hakan on 6/29/2015.
 */
public class MongoBulkService implements BulkService {

    private final Logger logger = LoggerFactory.getLogger(MongoBulkService.class);
    private final MongoCollection<Document> collection;

    public MongoBulkService(final MongoClient client, final YamlConfiguration config) {
        this.collection = client.getDatabase(config.getMisc().getDindex().getAs()).getCollection(config.getMisc().getCtype().getAs());
    }

    @Override
    public void proceed(List content) {
        try {
            logger.info("Transferring data began to mongodb.");
            collection.insertMany((List<Document>) content);
        } catch (Exception ex) {
            logger.debug(ex.getMessage(), ex);
        }
    }

    @Override
    public void dropDataSet() {
        if (collection.count() != 0) {
            String collectionName = collection.getNamespace().getCollectionName();
            collection.drop();
            logger.info(String.format("The current collection called %s was deleted.", collectionName));
        }
    }

    @Override
    public void close() {
        //no-op
    }

}


================================================
FILE: src/main/java/com/kodcu/util/codecs/CustomDateCodec.java
================================================
package com.kodcu.util.codecs;

import org.bson.BsonWriter;
import org.bson.codecs.DateCodec;
import org.bson.codecs.EncoderContext;

import java.text.SimpleDateFormat;
import java.util.Date;

/**
 * @author wwinder
 *         Created on: 5/27/16
 */
public class CustomDateCodec extends DateCodec {
    private final SimpleDateFormat formatter;

    public CustomDateCodec(String format) {
        formatter = new SimpleDateFormat(format);
    }

    @Override
    public void encode(final BsonWriter writer, final Date value, final EncoderContext encoderContext) {
        writer.writeString(formatter.format(value));
    }
}


================================================
FILE: src/main/java/com/kodcu/util/codecs/CustomLongCodec.java
================================================
package com.kodcu.util.codecs;

import org.bson.BsonWriter;
import org.bson.codecs.EncoderContext;
import org.bson.codecs.LongCodec;

/**
 * @author wwinder
 *         Created on: 5/27/16
 */
public class CustomLongCodec extends LongCodec {
    @Override
    public void encode(final BsonWriter writer, final Long value, final EncoderContext encoderContext) {
        writer.writeString(value.toString());
    }
}


================================================
FILE: src/main/resources/log4j2.properties
================================================
name=PropertiesConfig
property.filename = mongolastic
appenders = console, file

appender.console.type = Console
appender.console.name = STDOUT
appender.console.layout.type = PatternLayout
appender.console.layout.pattern = [%-5level] [%d{yyyy-MM-dd HH:mm:ss}] [%t] [%p]: - %msg%n
appender.file.type = File
appender.file.name = LOGFILE
appender.file.fileName=mongolastic.log
appender.file.layout.type=PatternLayout
appender.file.layout.pattern=[%-5level] [%d{yyyy-MM-dd HH:mm:ss}] [%t] [%p]: - %msg%n

loggers=file
logger.file.name=guru.springframework.blog.log4j2properties
logger.file.level = debug
logger.file.appenderRefs = file
logger.file.appenderRef.file.ref = LOGFILE

rootLogger.level = info
rootLogger.appenderRefs = stdout
rootLogger.appenderRef.stdout.ref = STDOUT

================================================
FILE: src/test/java/com/kodcu/test/TestMongoToElastic.java
================================================
package com.kodcu.test;

import com.kodcu.config.ElasticConfiguration;
import com.kodcu.config.FileConfiguration;
import com.kodcu.config.MongoConfiguration;
import com.kodcu.config.YamlConfiguration;
import com.kodcu.provider.ElasticToMongoProvider;
import com.kodcu.provider.MongoToElasticProvider;
import com.kodcu.provider.Provider;
import com.kodcu.service.BulkService;
import com.kodcu.service.ElasticBulkService;
import com.kodcu.service.MongoBulkService;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.IndicesAdminClient;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import java.util.Arrays;
import java.util.Objects;

import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;

/**
 * Created by Hakan on 9/8/2015.
 */
@RunWith(Parameterized.class)
public class TestMongoToElastic {

    private final FileConfiguration file;

    public TestMongoToElastic(final FileConfiguration file) {
        super();
        this.file = file;
    }

    @Parameterized.Parameters(name = "{index}: ({0})={1}")
    public static Iterable<Object[]> queries() throws Exception {
        String query = "misc:\n" +
                "    dindex:\n" +
                "        name: twitter\n" +
                "    ctype:\n" +
                "        name: tweets\n" +
                "    batch: 500\n" +
                "mongo:\n" +
                "    host: mongo\n" +
                "    port: 27017\n" +
                "elastic:\n" +
                "    host: es\n" +
                "    port: 9300";
        return Arrays.asList(new Object[][]{
                {new FileConfiguration("src/test/resources/conf1")},
                {new FileConfiguration("src/test/resources/conf3")},
                {new FileConfiguration(query)}
        });
    }

    @Test
    public void shouldCopyOneQueryToEsFromMongoDB() {
        YamlConfiguration config = file.getFileContent();
        assertThat(config, is(notNullValue()));

        if (Objects.isNull(config.getMongo().getAuth()))
            if (Objects.nonNull(System.getenv("MONGO_IP")))
                config.getMongo().setHost(System.getenv("MONGO_IP"));
            else
                config.getMongo().setHost("localhost");
        else {
            if (Objects.nonNull(System.getenv("MONGO_AUTH_IP")))
                config.getMongo().setHost(System.getenv("MONGO_AUTH_IP"));
            else
                return;
        }

        if (Objects.isNull(System.getenv("ES_IP")))
            config.getElastic().setHost("localhost");
        else
            config.getElastic().setHost(System.getenv("ES_IP"));


        ElasticConfiguration elastic = new ElasticConfiguration(config);
        MongoConfiguration mongo = new MongoConfiguration(config);

        BulkService bulkService = this.initializeBulkService(config, mongo, elastic);
        assertThat(bulkService, is(instanceOf(ElasticBulkService.class)));

        Provider provider = this.initializeProvider(config, mongo, elastic);
        assertThat(provider, is(instanceOf(MongoToElasticProvider.class)));

        provider.transfer(bulkService, config, () -> {
            bulkService.close();
            assertThat(provider.getCount(), equalTo(this.getCount(elastic, config)));
            elastic.closeNode();
            mongo.closeConnection();
        });
    }

    private Provider initializeProvider(YamlConfiguration config, MongoConfiguration mongo, ElasticConfiguration elastic) {
        if (config.getMisc().getDirection().equals("em")) {
            return new ElasticToMongoProvider(elastic, config);
        }
        return new MongoToElasticProvider(mongo.getMongoCollection(), config);
    }

    private BulkService initializeBulkService(YamlConfiguration config, MongoConfiguration mongo, ElasticConfiguration elastic) {
        if (config.getMisc().getDirection().equals("em")) {
            return new MongoBulkService(mongo.getClient(), config);
        }
        return new ElasticBulkService(config, elastic);
    }

    public long getCount(ElasticConfiguration elastic, YamlConfiguration config) {
        IndicesAdminClient admin = elastic.getClient().admin().indices();
        IndicesExistsRequestBuilder builder = admin.prepareExists(config.getMisc().getDindex().getAs());
        assertThat(builder.execute().actionGet().isExists(), is(true));

        elastic.getClient().admin().indices().flush(new FlushRequest(config.getMisc().getDindex().getAs())).actionGet();

        SearchResponse response = elastic.getClient().prepareSearch(config.getMisc().getDindex().getAs())
                .setTypes(config.getMisc().getCtype().getAs())
                .setSearchType(SearchType.QUERY_THEN_FETCH)
                .setSize(0)
                .execute().actionGet();
        long count = response.getHits().getTotalHits();
        return count;
    }
}


================================================
FILE: src/test/java/com/kodcu/test/TestMongolasticQueries.java
================================================
package com.kodcu.test;

import com.kodcu.config.FileConfiguration;
import com.kodcu.config.YamlConfiguration;
import com.kodcu.config.structure.Elastic;
import com.kodcu.config.structure.Misc;
import com.kodcu.config.structure.Mongo;
import com.kodcu.config.structure.Namespace;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;

import java.util.Arrays;

import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.junit.Assert.assertThat;

/**
 * Created by Hakan on 8/24/2015.
 */
@RunWith(Parameterized.class)
public class TestMongolasticQueries {

    private final FileConfiguration config;
    private final YamlConfiguration expected;

    public TestMongolasticQueries(final FileConfiguration config, final YamlConfiguration expected) {
        super();
        this.config = config;
        this.expected = expected;
    }

    @Parameterized.Parameters(name = "{index}: ({0})={1}")
    public static Iterable<Object[]> queries() throws Exception {
        return Arrays.asList(new Object[][]{
                {new FileConfiguration("src/test/resources/conf1"), createQueryConfiguration1()},
                {new FileConfiguration("src/test/resources/conf2"), createQueryConfiguration2()}
        });
    }

    private static YamlConfiguration createQueryConfiguration1() {
        YamlConfiguration query = new YamlConfiguration();

        Misc misc = new Misc();
        Namespace db = new Namespace();
        db.setName("twitter");
        db.setAs("kodcu");
        misc.setDindex(db);
        Namespace c = new Namespace();
        c.setName("tweets");
        c.setAs("tweets");
        misc.setCtype(c);
        misc.setBatch(300);
        query.setMisc(misc);

        Mongo mongod = new Mongo();
        mongod.setHost("mongo");
        mongod.setPort(27017);
        mongod.setQuery("{}");
        query.setMongo(mongod);

        Elastic es = new Elastic();
        es.setHost("es");
        es.setPort(9300);
        query.setElastic(es);
        return query;
    }

    private static YamlConfiguration createQueryConfiguration2() {
        YamlConfiguration query = new YamlConfiguration();

        Misc misc = new Misc();
        Namespace db = new Namespace();
        db.setName("twitter");
        db.setAs("twitter");
        misc.setDindex(db);
        Namespace c = new Namespace();
        c.setName("tweets");
        c.setAs("posts");
        misc.setCtype(c);
        misc.setDirection("em");
        misc.setBatch(200);
        query.setMisc(misc);

        Mongo mongod = new Mongo();
        mongod.setHost("127.0.0.1");
        mongod.setPort(27017);
        query.setMongo(mongod);

        Elastic es = new Elastic();
        es.setHost("127.0.0.1");
        es.setPort(9300);
        query.setElastic(es);
        return query;
    }

    @Test
    public void shouldProceedQueries() {
        YamlConfiguration actual = config.getFileContent();
        assertThat(actual, notNullValue());
        assertThat(actual.getMongo().getQuery(), is(expected.getMongo().getQuery()));
        assertThat(actual.getMongo().getHost(), is(expected.getMongo().getHost()));
        assertThat(actual.getMongo().getPort(), is(expected.getMongo().getPort()));
        assertThat(actual.getElastic().getHost(), is(expected.getElastic().getHost()));
        assertThat(actual.getElastic().getPort(), is(expected.getElastic().getPort()));
        assertThat(actual.getMisc().getDindex().getName(), is(expected.getMisc().getDindex().getName()));
        assertThat(actual.getMisc().getDindex().getAs(), is(expected.getMisc().getDindex().getAs()));
        assertThat(actual.getMisc().getCtype().getName(), is(expected.getMisc().getCtype().getName()));
        assertThat(actual.getMisc().getCtype().getAs(), is(expected.getMisc().getCtype().getAs()));
    }
}


================================================
FILE: src/test/resources/conf1
================================================
misc:
    dindex:
        name: twitter
        as: kodcu
    ctype:
        name: tweets
        as: tweets
    batch: 300
#    direction: me
mongo:
    host: mongo
    port: 27017
elastic:
    host: es
    port: 9300

================================================
FILE: src/test/resources/conf2
================================================
{
	"misc": {
		"dindex": {
			"name": "twitter"
		},
		"ctype": {
			"name": "tweets",
			"as": "posts"
		},
		"direction": "em"
	},
	"mongo": {
		"host": "127.0.0.1",
		"port": 27017
	},
	"elastic": {
		"host": "127.0.0.1",
		"port": 9300
	}
}

================================================
FILE: src/test/resources/conf3
================================================
misc:
    dindex:
        name: twitter
        as: twt
    ctype:
        name: tweets
        as: posts
    batch: 500
#    direction: me
mongo:
    host: mongo_auth
    port: 27017
    auth:
        user: hakan
        pwd: "1234"
        source: admin
        mechanism: scram-sha-1
elastic:
    host: es
    port: 9300