Remove messages after reading them (safely)

feedbackloop/main.py

@@ -7,7 +7,7 @@ from .app import app, db
 # Import all views by side effect.
 from . import views
 from .model import FeedbackEntry
-from .parse import scan_imap
+from .parse import MailScanner
 
 
 def create_app(config=None):
@@ -28,11 +28,12 @@ manager = Manager(create_app)
 def ingest(unseen, limit):
     """Read and ingest new messages from the IMAP mailbox."""
     with app.app_context():
-        for entry in scan_imap(
+        with MailScanner(
                 app.config['IMAP_SERVER'],
                 app.config['IMAP_USERNAME'],
                 app.config['IMAP_PASSWORD'],
-                unseen, limit):
+                app.config.get('IMAP_REMOVE_MESSAGES', True)) as scanner:
+            for entry in scanner.scan(unseen, limit):
                 db.session.add(entry)
         db.session.commit()
 

feedbackloop/parse.py

@@ -55,33 +55,62 @@ def _hdr(hdr_list, name):
             return v
 
 
-def _scan_mailbox(server, username, password, unseen=False, limit=None):
-    conn = imaplib.IMAP4_SSL(server)
-    conn.socket().settimeout(30)
-    i = 0
+
+class MailScanner():
+    """Read messages from a remote IMAP folder and remove them.
+
+    Operates as a context manager, messages scanned are only removed
+    if the inner block executes successfully.
+    """
+
+    def __init__(self, server, username, password, remove=True):
+        self._server = server
+        self._username = username
+        self._password = password
+        self._remove = remove
+        self._to_delete = []
+        self._conn = None
+
+    def __enter__(self):
+        self._conn = imaplib.IMAP4_SSL(self._server)
+        self._conn.socket().settimeout(30)
+        try:
+            self._conn.login(self._username, self._password)
+            self._conn.select('INBOX')
+        except:
+            try:
+                self._conn.close()
+            except:
+                pass
+            raise
+        return self
+
+    def __exit__(self, exc_type, value, traceback):
+        if self._remove and not exc_type:
+            for uid in self._to_delete:
+                self._conn.uid('store', uid.encode('utf-8'), '+FLAGS', '\\Deleted')
+            self._conn.expunge()
         try:
-        conn.login(username, password)
-        conn.select('INBOX')
-        res, data = conn.uid('search', None, '(UNSEEN)' if unseen else 'ALL')
+            self._conn.close()
+        except:
+            pass
+
+    def _scan_mailbox(self, unseen=False, limit=None):
+        i = 0
+        res, data = self._conn.uid('search', None, '(UNSEEN)' if unseen else 'ALL')
         if res != 'OK':
             raise IMAPError(res)
         for uid in data[0].decode('utf-8').split():
-            res, response = conn.uid('fetch', uid, '(RFC822)')
+            res, response = self._conn.uid('fetch', uid, '(RFC822)')
             if res != 'OK':
                 raise IMAPError(res)
-            yield response[0][1]
+            yield uid, response[0][1]
             i += 1
             if limit and i > limit:
                 break
-    finally:
-        try:
-            conn.close()
-        except:
-            pass
-
 
-def scan_imap(server, user, password, unseen, limit=None):
-    for raw_msg in _scan_mailbox(server, user, password, unseen, limit):
+    def scan(self, unseen, limit=None):
+        for uid, raw_msg in self._scan_mailbox(unseen, limit):
             try:
 
                 # Parse the ARF message body.
@@ -148,6 +177,7 @@ def scan_imap(server, user, password, unseen, limit=None):
                     entry.message = raw_msg
                 print(f'original from: {original_from}, list={is_list}')
                 yield entry
+                self._to_delete.append(uid)
 
             except Exception as e:
                 print(f'error: {e}')
@@ -162,7 +192,8 @@ def main():
     parser.add_argument('--unseen', action='store_true')
     args = parser.parse_args()
 
-    for entry in scan_imap(args.server, args.user, args.password, args.unseen, 20):
+    with MailScanner(args.server, args.user, args.password, False) as scanner:
+        for entry in scanner.scan(args.unseen, 20):
             print(entry)
 
 

feedbackloop/views.py

 from flask import render_template, abort
-from sqlalchemy import func
+from sqlalchemy import func, text
 from .app import app, db
 from .model import FeedbackEntry
 
 
-@app.route('/report/:report_id')
+@app.route('/report/<report_id>')
 def show_report(report_id):
     report = FeedbackEntry.query.get(report_id)
     if not report:
@@ -12,10 +12,11 @@ def show_report(report_id):
     return render_template('report.html', report=report)
 
 
-@app.route('/by_sender/:sender')
+@app.route('/by_sender/<sender>')
 def by_sender(sender):
     reports = FeedbackEntry.query.filter(
-        FeedbackEntry.sender == sender).order_by('timestamp desc')
+        FeedbackEntry.sender == sender).order_by(
+            FeedbackEntry.timestamp.desc())
     if not reports:
         abort(404)
     return render_template('sender.html',
@@ -26,15 +27,19 @@ def by_sender(sender):
 @app.route('/')
 def index():
     top_user_senders = db.session.query(
-        func.count(FeedbackEntry.id)).filter(
-            FeedbackEntry.is_list is False).group_by(
-                FeedbackEntry.sender).limit(20)
+        FeedbackEntry.sender,
+        func.count(FeedbackEntry.id).label('count')).filter(
+            FeedbackEntry.is_list == False).group_by(
+                FeedbackEntry.sender).order_by(text('count DESC')).limit(20)
     top_list_senders = db.session.query(
-        func.count(FeedbackEntry.id)).filter(
-            FeedbackEntry.is_list is True).group_by(
-                FeedbackEntry.sender).limit(20)
+        FeedbackEntry.sender,
+        func.count(FeedbackEntry.id).label('count')).filter(
+            FeedbackEntry.is_list == True).group_by(
+                FeedbackEntry.sender).order_by(text('count DESC')).limit(20)
     top_reporters = db.session.query(
-        func.count(FeedbackEntry.id)).group_by(FeedbackEntry.reporter)
+        FeedbackEntry.reporter,
+        func.count(FeedbackEntry.id).label('count')).group_by(
+            FeedbackEntry.reporter).order_by(text('count DESC'))
 
     return render_template('index.html',
                            top_user_senders=top_user_senders,