diff --git a/server/djrandom/model/processor.py b/server/djrandom/model/processor.py index 567f6dc3f196bbae2b0ae48b9160de7afd9efa3a..c72211d3f019940dfa7926c3b3f02007e792c2a3 100644 --- a/server/djrandom/model/processor.py +++ b/server/djrandom/model/processor.py @@ -5,6 +5,8 @@ from djrandom.database import Session log = logging.getLogger(__name__) +COMMIT_EVERY = 50 + class Processor(object): """Base class to perform operations on database objects. @@ -24,7 +26,6 @@ class Processor(object): n = 0 items = self.query() for item in items: - n += 1 try: self.process(item) Session.add(item) @@ -32,6 +33,9 @@ class Processor(object): log.error( 'Unexpected exception processing item %s:\n %s' % ( item, traceback.format_exc())) + n += 1 + if (n % COMMIT_EVERY == 0) and not self._dry_run: + Session.commit() if n > 0 and not self._dry_run: Session.commit() self.commit() @@ -53,7 +57,7 @@ class Processor(object): try: n = self.full_scan() except Exception, e: - log.error('Temporary error: %s' % e) + log.exception('Temporary error') Session.remove() if run_once: break