[elasticsearch] elasticsearch 10000개 이상 데이터 조회 python

es에서 데이터를 10000개 이상 조회하려면 scroll id를 사용하거나 

 

elasticsearch helpers의 scan 을 사용해야한다.

 

helpers.scan

elasticsearch.helpers.scan(conn, scroll='10m', index=index, doc_type='_doc', size=10000, query=query)

 

es.scroll

def scroll_API(index, body):
    result = []

    _KEEP_ALIVE_LIMIT='30s'

    # Initialize the scroll
    page = es.search(index=index
                    ,body=body
                    ,scroll=_KEEP_ALIVE_LIMIT
                    ,size=10000
                    ,track_total_hits=True
                    )
    sid = page['_scroll_id']
    scroll_size = page['hits']['total']['value']

    # Start scrolling
    result += page['hits']['hits']


    while (scroll_size > 0):
        print('Scrolling...')
        page = es.scroll(scroll_id=sid, scroll=_KEEP_ALIVE_LIMIT)

        # Update the scroll ID
        sid = page['_scroll_id']

        # Get the number of results that we returned in the last scroll
        scroll_size = len(page['hits']['hits'])
        print("scroll size: ", str(scroll_size))
        result += page['hits']['hits']

    es.clear_scroll(body={'scroll_id': [sid]}, ignore=(404, ))
    return result

 

 

 

참고

https://stackoverflow.com/questions/46604207/elasticsearch-scroll