commit more often

server/djrandom/model/processor.py

@@ -5,6 +5,8 @@ from djrandom.database import Session
 
 log = logging.getLogger(__name__)
 
+COMMIT_EVERY = 50
+
 
 class Processor(object):
     """Base class to perform operations on database objects.
@@ -24,7 +26,6 @@ class Processor(object):
         n = 0
         items = self.query()
         for item in items:
-            n += 1
             try:
                 self.process(item)
                 Session.add(item)
@@ -32,6 +33,9 @@ class Processor(object):
                 log.error(
                     'Unexpected exception processing item %s:\n %s' % (
                         item, traceback.format_exc()))
+            n += 1
+            if (n % COMMIT_EVERY == 0) and not self._dry_run:
+                Session.commit()
         if n > 0 and not self._dry_run:
             Session.commit()
             self.commit()
@@ -53,7 +57,7 @@ class Processor(object):
             try:
                 n = self.full_scan()
             except Exception, e:
-                log.error('Temporary error: %s' % e)
+                log.exception('Temporary error')
             Session.remove()
             if run_once:
                 break