Extract data from elasticsearch and store into mongodb

Working with Data Connectors & Integrations
1

Hi,
I’m looking for an option to extract data from Elasticsearch and store into the mongodb using some python script. Please help me with that.

2

Hi ,
Here is my solution, may be any one can comment, if anyway I can improve this code or how I can query the data from an index pattern

from elasticsearch import Elasticsearch
from elasticsearch.helpers import scan
import pandas as pd
import pymongo
from pymongo import MongoClient


esClient = Elasticsearch('http://localhost:9200/')

def get_data_from_elastic():
    query = {
        "query": {
            "match": {
                 "state": "failed"
            }
        }
    }
    res=scan(client = esClient,
        query = query,
        scroll = '1m',
        index = 'test-2022-11-09',
        raise_on_error=True,
        preserve_order=False,
        clear_scroll=True)
    
    result = list(res)
    temp = []
    
    for hit in result:
        temp.append(hit['_source'])
    
    df = pd.DataFrame(temp)
    return df
df = get_data_from_elastic()
print(df.head())

client = MongoClient('mongodb://admin:vbox@10.20.30.40:27001/admin')
db = client['elasticDB']
elastic_data = db['elastic_data']
    
df.reset_index(inplace=True)
data_dict = df.to_dict("records")
elastic_data.insert_many(data_dict)