From 95ee6a06727e885b81c66d5cb4f7f55ec303b964 Mon Sep 17 00:00:00 2001
From: ale <ale@incal.net>
Date: Fri, 27 Jan 2012 14:29:42 +0100
Subject: [PATCH] commit more often
---
server/djrandom/model/processor.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/server/djrandom/model/processor.py b/server/djrandom/model/processor.py
index 567f6dc..c72211d 100644
--- a/server/djrandom/model/processor.py
+++ b/server/djrandom/model/processor.py
@@ -5,6 +5,8 @@ from djrandom.database import Session
log = logging.getLogger(__name__)
+COMMIT_EVERY = 50
+
class Processor(object):
"""Base class to perform operations on database objects.
@@ -24,7 +26,6 @@ class Processor(object):
n = 0
items = self.query()
for item in items:
- n += 1
try:
self.process(item)
Session.add(item)
@@ -32,6 +33,9 @@ class Processor(object):
log.error(
'Unexpected exception processing item %s:\n %s' % (
item, traceback.format_exc()))
+ n += 1
+ if (n % COMMIT_EVERY == 0) and not self._dry_run:
+ Session.commit()
if n > 0 and not self._dry_run:
Session.commit()
self.commit()
@@ -53,7 +57,7 @@ class Processor(object):
try:
n = self.full_scan()
except Exception, e:
- log.error('Temporary error: %s' % e)
+ log.exception('Temporary error')
Session.remove()
if run_once:
break
--
GitLab