use a single "state" attribute for the MP3 table

server/djrandom/fingerprint/dedup.py

@@ -57,7 +57,7 @@ class DeDuper(object):
         fp.erase_database(local=True, really_delete=True)
         mp3s = MP3.query.filter(
             (MP3.artist == u'bonobo') 
-            & (MP3.ready == True) & (MP3.error == False) 
+            & (MP3.state == MP3.READY)
             & (MP3.echoprint_fp != None))
         for mp3 in mp3s:
             code = self._generate_code_json(mp3.echoprint_fp, mp3.sha1)
@@ -140,6 +140,18 @@ class DeDuper(object):
         """Perform best duplicate selection and remove dupes from db."""
         log.debug('remove_dupes(%s)' % ','.join(hashes))
 
+        # Compute 'score' for each song and sort them.
+        scores = []
+        mp3s = MP3.query.filter(MP3.sha1 in hashes)
+        for mp3 in mp3s:
+            scores.append((self._get_song_score(mp3), mp3.sha1))
+        scores.sort(key=lambda x: x[0])
+        best_song = scores[0][1]
+        log.debug('remove_dupes: best song is %s' % best_song)
+
+        # Remove all the other songs.
+        songs_to_remove = [x for x in hashes if x != best_song]
+        
 
 
 def run_deduper(db_url):

server/djrandom/fingerprint/fingerprint.py

@@ -31,7 +31,8 @@ class Fingerprinter(object):
     def compute_fingerprints(self, run_once):
         """Compute fingerprints of new files."""
         while True:
-            mp3 = MP3.query.filter(MP3.echoprint_fp == None
+            mp3 = MP3.query.filter((MP3.state == MP3.READY)
+                                   & (MP3.echoprint_fp == None)
                                    ).limit(1).first()
             if not mp3:
                 if run_once:

server/djrandom/frontend/api_views.py

@@ -28,8 +28,7 @@ def artist_albums_json(artist):
 @app.route('/json/album/<artist>/<album>')
 @require_auth
 def album_songs_json(artist, album):
-    songs = [x.to_dict() for x in MP3.query.filter_by(
-            error=False, ready=True, artist=artist, album=album)]
+    songs = [x.to_dict() for x in MP3.get_songs_for_album(artist, album)]
     return jsonify(songs=songs)
 
 

server/djrandom/model/mp3.py

@@ -13,9 +13,13 @@ class MP3(Base):
 
     __tablename__ = 'mp3'
 
+    INCOMING = 'I'
+    READY = 'R'
+    ERROR = 'E'
+    DUPLICATE = 'D'
+
     sha1 = Column(String(40), primary_key=True)
-    ready = Column(Boolean, default=False, index=True)
-    error = Column(Boolean, default=False, index=True)
+    state = Column(String(1), default=INCOMING, index=True)
     path = Column(String(1024))
     size = Column(Integer())
     artist = Column(Unicode(256))
@@ -25,37 +29,47 @@ class MP3(Base):
     uploaded_at = Column(DateTime())
     play_count = Column(Integer(), default=0)
     echoprint_fp = deferred(Column(Text()))
+    duplicate_of = Column(String(40))
 
     def __init__(self, **kw):
         for k, v in kw.items():
             setattr(self, k, v)
 
     def to_dict(self):
-        return {'title': self.title,
+        data = {'title': self.title,
                 'artist': self.artist,
                 'album': self.album,
                 'genre': self.genre,
                 'sha1': self.sha1,
                 'size': self.size,
                 'uploaded_at': self.uploaded_at}
+        if self.duplicate:
+            data['duplicate_of'] = self.duplicate_of
+        return data
 
     @classmethod
     def last_uploaded(cls, n=10):
         """Return the N last uploaded songs."""
-        return cls.query.filter_by(ready=True).order_by(
+        return cls.query.filter_by(state=cls.READY).order_by(
             desc(cls.uploaded_at)).limit(n)
 
     @classmethod
     def get_random_songs(cls, n=10):
         """Return N completely random songs."""
         results = []
-        num_songs = cls.query.filter_by(ready=True).count()
-        for idx in xrange(n):
-            song = cls.query.filter_by(ready=True).limit(1).offset(
-                random.randint(0, num_songs - 1)).one()
-            results.append(song)
+        num_songs = cls.query.filter_by(state=cls.READY).count()
+        fraction = float(n) / num_songs
+        while len(results) < n:
+            tmprows = Session.query(cls.sha1).filter(func.rand() < fraction).limit(n)
+            for row in tmprows:
+                results.append(row[0])
         return results
 
+    @classmethod
+    def get_songs_for_album(cls, artist, album):
+        return cls.query.filter_by(
+            state=cls.READY, artist=artist, album=album)
+
 
 class PlayLog(Base):
 

server/djrandom/receiver/receiver.py

@@ -32,6 +32,7 @@ def check(sha1):
 
 def _upload_mp3(incoming_fd, sha1):
     mp3 = MP3(path=utils.generate_path(storage_root, sha1),
+              state=MP3.INCOMING,
               sha1=sha1,
               uploaded_at=datetime.now())
     with open(mp3.path, 'w') as fd:

server/djrandom/scanner/scanner.py

@@ -27,7 +27,7 @@ class Scanner(object):
     def scan_db(self, run_once):
         """Scan the database for new files."""
         while True:
-            mp3 = MP3.query.filter_by(ready=False, error=False
+            mp3 = MP3.query.filter_by(state=MP3.INCOMING
                                       ).limit(1).first()
             if not mp3:
                 if run_once:
@@ -39,10 +39,10 @@ class Scanner(object):
             log.info('processing %s' % mp3.sha1)
             try:
                 self.process(mp3)
-                mp3.ready = True
+                mp3.state = MP3.READY
             except Exception, e:
                 log.error(traceback.format_exc())
-                mp3.error = True
+                mp3.state = MP3.ERROR
             Session.add(mp3)
             Session.commit()