Commit 610af10f authored by ale's avatar ale

add statistical analysis graphs with faceted search results

parent a3171cc6
......@@ -197,15 +197,28 @@ class LogStore(object):
**search_args)
return search
def search(self, query_str, time_range, size=100, start=0):
log.info('search: "%s", range=%s, start=%d',
query_str, time_range, start)
def search(self, query_str, time_range, size=100, start=0, facets=None):
log.debug('search: "%s", range=%s, start=%d, facets=%s',
query_str, time_range, start, facets)
search = self._make_search(query_str, time_range,
start=start, size=size)
results = self.conn.search(search,
search_type='query_then_fetch')
total = results['hits']['total']
return [x['_source'] for x in results['hits']['hits']], total
if facets:
for f, fsize in facets:
if f == 'timestamp':
search.facet.facets.append(
pyes.facets.HistogramFacet(
'timestamp',
field='timestamp',
interval=86400 * 1000000))
else:
search.facet.add_term_facet(f, size=fsize)
response = self.conn.search(search,
search_type='query_then_fetch')
total = response['hits']['total']
docs = [x['_source'] for x in response['hits']['hits']]
result_facets = response.get('facets') if facets else {}
return docs, total, result_facets, response['took']
def scan(self, query_str, time_range, size=100):
log.info('scan: "%s"', query_str)
......
......@@ -58,10 +58,12 @@ def do_search(lens, opts, args):
count = 0
page_size = 400
while True:
results, total = lens.search(query, time_range, start=count, size=page_size)
results, total, facet_data, elapsed = lens.search(
query, time_range, start=count, size=page_size)
if not results:
break
log.debug('search: %d results (%d/%d)', len(results), count, total)
log.debug('search: %d results (%d/%d), %g secs', len(results),
count, total, elapsed)
for doc in results:
sys.stdout.write(utils.format_log(doc).encode('utf-8', 'replace'))
count += len(results)
......
This diff is collapsed.
......@@ -14,30 +14,68 @@
}
#log_pagination_div {
display: none;
font-size: 85%;
padding-bottom: 3px;
padding-left: 30px;
padding-right: 30px;
}
#log_pagination_prev {
cursor: pointer;
}
#log_pagination_next {
cursor: pointer;
}
#query_time {
float: right;
margin-right: 30px;
color: #aaa;
}
#show_stats_btn {
display: block;
float: right;
margin-bottom: -3px;
padding-left: 20px;
padding-right: 5px;
padding-top: 2px;
border: 1px solid black;
font-weight: bold;
}
.btn_open {
background: url(img/arrow_open.png) top left no-repeat;
}
.btn_closed {
background: url(img/arrow_closed.png) top left no-repeat;
}
#log_stats {
display: none;
width: 100%;
border-top: 1px solid #666;
padding-top: 3px;
padding-bottom: 3px;
}
#log_viewer {
border-top: 1px solid #666;
padding-top: 3px;
font-family: 'Ubuntu Mono', monospace;
word-wrap: break-word;
}
#log_viewer a {
color: #09f;
text-decoration: none;
padding-left: 1px;
padding-right: 1px;
}
#log_viewer a:hover {
background-color: #009;
color: white;
......@@ -49,7 +87,20 @@
padding-right: 40px;
background: url(/static/rstar32.gif) no-repeat top right;
}
#error {
display: block;
color: red;
}
.chart {
float: left;
margin: 5px;
}
.chart h4 {
font-weight: normal;
padding: 0;
margin: 0;
}
......@@ -56,6 +56,8 @@ lens.Lens = function(config) {
};
this.results_view_id = _merge('log_viewer_id', '#log_viewer');
this.results_view_elem = $(this.results_view_id);
this.stats_view_id = _merge('log_stats_id', '#log_stats_inner');
this.stats_view_elem = $(this.stats_view_id);
this.query_field_elem = $(_merge('query_field_id', '#query_field'));
this.pagination_div_elem = $(_merge('pagination_div_id', '#log_pagination_div'));
this.pagination_title_elem = $(_merge('pagination_title_id', '#log_pagination_title'));
......@@ -71,6 +73,8 @@ lens.Lens = function(config) {
this.log_template = $.template(null, $(log_template_id));
var error_template_id = _merge('error_template_id', '#error_template');
this.error_template = $.template(null, $(error_template_id));
var chart_template_id = _merge('chart_template_id', '#chart_template');
this.chart_template = $.template(null, $(chart_template_id));
// Reset internal state.
this.resetDefaultQueryParams();
......@@ -228,7 +232,21 @@ lens.Lens.prototype.renderPagination = function(logs) {
this.pagination_div_elem.show();
};
lens.Lens.prototype.render = function(logs) {
lens.Lens.prototype.renderChart = function(chart_attr, chart_title, chart_data) {
$.tmpl(this.chart_template,
{name: chart_attr, title: chart_title}
).appendTo(this.stats_view_id);
Donut(this, chart_attr, 'chart_graph_' + chart_attr).data(chart_data).draw();
};
lens.Lens.prototype.renderHistogram = function(chart_attr, chart_title, chart_data) {
$.tmpl(this.chart_template,
{name: chart_attr, title: chart_title}
).appendTo(this.stats_view_id);
Timeline('chart_graph_' + chart_attr).data(chart_data).draw();
};
lens.Lens.prototype.render = function(logs, facets, elapsed) {
// Render the logs.
if (logs.length > 0) {
this.renderLogs(logs);
......@@ -238,6 +256,23 @@ lens.Lens.prototype.render = function(logs) {
// Render pagination elements.
this.renderPagination(logs);
// Render stats.
this.stats_view_elem.html('');
if (facets.host) {
this.renderChart('host', 'Top 5 hosts', facets.host.terms);
}
if (facets.program) {
this.renderChart('program', 'Top 5 programs', facets.program.terms);
}
if (facets.timestamp) {
this.renderHistogram('timestamp', 'Time distribution',
facets.timestamp.entries);
}
this.stats_view_elem.show();
// Show query time.
$('#query_time').text('query time: ' + elapsed + ' ms');
};
/**
......@@ -267,6 +302,7 @@ lens.Lens.prototype.refresh = function() {
start: this.offset,
page_size: this.page_size,
time_range: time_range_str,
facets: 'timestamp'
},
dataType: 'json',
success: function(data, status, jqxhr) {
......@@ -274,7 +310,7 @@ lens.Lens.prototype.refresh = function() {
lensobj.ajaxError(jqxhr, 'query_error', data.error);
} else {
lensobj.total_results = data.total_results;
lensobj.render(data.results);
lensobj.render(data.results, data.facets, data.elapsed);
lensobj.clearLoading();
lensobj.saveState();
}
......
......@@ -9,6 +9,8 @@
<script type="text/javascript" src="/static/strftime.min.js"></script>
<script type="text/javascript" src="/static/jquery-1.7.1.min.js"></script>
<script type="text/javascript" src="/static/jquery.tmpl.min.js"></script>
<script type="text/javascript" src="/static/protovis.min.js"></script>
<script type="text/javascript" src="/static/graphs.js"></script>
<script type="text/javascript" src="/static/lens2.js?v=123"></script>
<script type="text/javascript">
......@@ -24,6 +26,11 @@ $(document).ready(function() {
return false;
});
$('#show_stats_btn').click(
function() {
$('#log_stats').toggle('fast');
$(this).toggleClass('btn_open btn_closed');
});
});
</script>
......@@ -41,8 +48,8 @@ $(document).ready(function() {
</form>
</div>
<div id="log_pagination_div">
<!-- Note that the time-based navigation logic is reversed,
since we retrieve results in timestamp-descending order. -->
<a id="show_stats_btn" class="btn_closed">Stats</a>
<div id="query_time"></div>
<a id="log_pagination_prev" style="display:none;">
&lt;-- Previous
</a>
......@@ -53,7 +60,13 @@ $(document).ready(function() {
</div>
</div>
<div id="log_viewer">
<div id="log_container">
<div id="log_stats">
<b>Result set analysis</b><br>
<div id="log_stats_inner"></div>
<div style="clear:both;"></div>
</div>
<div id="log_viewer"></div>
</div>
{% raw %}
......@@ -72,6 +85,13 @@ $(document).ready(function() {
ERROR: ${error}<br>
<span class="error_detail">${errorThrown}</span>
</div>
<div id="chart_template" style="display:none;">
<div class="chart">
<h4>${title}:</h4>
<div id="chart_graph_${name}"></div>
</div>
</div>
{% endraw %}
</body>
......
......@@ -9,20 +9,30 @@ log = logging.getLogger(__name__)
@api_app.route('/search', methods=['POST'])
def api_search():
facets = [('host', 5), ('program', 5)]
try:
query = request.form['q']
start = int(request.form.get('start', 0))
page_size = int(request.form.get('page_size', 20))
time_range = utils.parse_time_range(
request.form.get('time_range', '1d'))
for facet_spec in request.form.get('facets', '').split():
if ':' in facet_spec:
facet_term, facet_size = facet_spec.split(':')
else:
facet_term, facet_size = facet_spec, 5
facets.append((facet_term, int(facet_size)))
except (KeyError, ValueError):
log.exception('search not parsed')
abort(400)
try:
results, total = current_app.logstore.search(
query, time_range, start=start, size=page_size)
return jsonify(results=results, total_results=total)
results, total, facet_data, elapsed = current_app.logstore.search(
query, time_range, start=start, size=page_size, facets=facets)
return jsonify(results=results,
total_results=total,
facets=facet_data,
elapsed=elapsed)
except Exception, e:
return jsonify(error=str(e))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment