Commit 7d686da7 authored by ale's avatar ale

fix search pagination

parent e97bef13
......@@ -192,16 +192,14 @@ class LogStore(object):
**search_args)
return search
def search(self, query_str, time_range, size=100, scroll_id=None):
def search(self, query_str, time_range, size=100, start=0):
log.info('search: "%s"', query_str)
search_args = {'size': size,
'scroll': '30m'}
if scroll_id:
search_args['scroll_id'] = scroll_id
search = self._make_search(query_str, time_range, **search_args)
results = self.conn.search(search)
scroll_id = results['scroll_id']
return [x['_source'] for x in results['hits']['hits']], scroll_id
search = self._make_search(query_str, time_range,
start=start, size=size)
results = self.conn.search(search,
search_type='query_then_fetch')
total = results['hits']['total']
return [x['_source'] for x in results['hits']['hits']], total
def scan(self, query_str, time_range, size=100):
log.info('scan: "%s"', query_str)
......
......@@ -55,15 +55,18 @@ def do_search(lens, opts, args):
if opts.time_range:
time_range = utils.parse_time_range(opts.time_range)
scroll_id = None
count = 0
page_size = 400
while True:
results, scroll_id = lens.search(query, time_range,
scroll_id=scroll_id)
results, total = lens.search(query, time_range, start=count, size=page_size)
if not results:
break
log.debug('search: %d results, scroll_id=%s', len(results),
scroll_id)
log.debug('search: %d results (%d/%d)', len(results), count, total)
for doc in results:
sys.stdout.write(utils.format_log(doc).encode('utf-8', 'replace'))
count += len(results)
if count >= total:
break
sys.stdout.flush()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment