diff --git a/client/osx-ui/DJRandomUploader/Preferences.h b/client/osx-ui/DJRandomUploader/Preferences.h
new file mode 100644
index 0000000000000000000000000000000000000000..0afa5866ea96aa86279b3142dbc9d25e47815086
--- /dev/null
+++ b/client/osx-ui/DJRandomUploader/Preferences.h
@@ -0,0 +1,17 @@
+//
+// Preferences.h
+// DJRandomUploader
+//
+// Created by Alessandro Preite Martinez on 01/11/2011.
+// Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import <Cocoa/Cocoa.h>
+
+
+@interface Preferences : NSWindowController {
+@private
+
+}
+
+@end
diff --git a/client/osx-ui/DJRandomUploader/Preferences.m b/client/osx-ui/DJRandomUploader/Preferences.m
new file mode 100644
index 0000000000000000000000000000000000000000..81632bf9b52b658991097e90adac55d4531cf32e
--- /dev/null
+++ b/client/osx-ui/DJRandomUploader/Preferences.m
@@ -0,0 +1,36 @@
+//
+// Preferences.m
+// DJRandomUploader
+//
+// Created by Alessandro Preite Martinez on 01/11/2011.
+// Copyright 2011 __MyCompanyName__. All rights reserved.
+//
+
+#import "Preferences.h"
+
+
+@implementation Preferences
+
+- (id)initWithWindow:(NSWindow *)window
+{
+ self = [super initWithWindow:window];
+ if (self) {
+ // Initialization code here.
+ }
+
+ return self;
+}
+
+- (void)dealloc
+{
+ [super dealloc];
+}
+
+- (void)windowDidLoad
+{
+ [super windowDidLoad];
+
+ // Implement this method to handle any initialization after your window controller's window has been loaded from its nib file.
+}
+
+@end
diff --git a/client/osx-ui/DJRandomUploader/Preferences.xib b/client/osx-ui/DJRandomUploader/Preferences.xib
new file mode 100644
index 0000000000000000000000000000000000000000..7f1b38acb1e5b296ed183682d67956771dbfbb34
--- /dev/null
+++ b/client/osx-ui/DJRandomUploader/Preferences.xib
@@ -0,0 +1,162 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<archive type="com.apple.InterfaceBuilder3.Cocoa.XIB" version="7.050000190734863">
+ <data>
+ <int key="IBDocument.SystemTarget">1060</int>
+ <string key="IBDocument.SystemVersion">10A216</string>
+ <string key="IBDocument.InterfaceBuilderVersion">708</string>
+ <string key="IBDocument.AppKitVersion">994.4</string>
+ <string key="IBDocument.HIToolboxVersion">404.00</string>
+ <object class="NSMutableDictionary" key="IBDocument.PluginVersions">
+ <string key="NS.key.0">com.apple.InterfaceBuilder.CocoaPlugin</string>
+ <string key="NS.object.0">708</string>
+ </object>
+ <object class="NSMutableArray" key="IBDocument.EditedObjectIDs">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ </object>
+ <object class="NSArray" key="IBDocument.PluginDependencies">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ <string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+ </object>
+ <object class="NSMutableDictionary" key="IBDocument.Metadata">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ <object class="NSArray" key="dict.sortedKeys">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ </object>
+ <object class="NSMutableArray" key="dict.values">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ </object>
+ </object>
+ <object class="NSMutableArray" key="IBDocument.RootObjects" id="1000">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ <object class="NSCustomObject" id="1001">
+ <string key="NSClassName">NSObject</string>
+ </object>
+ <object class="NSCustomObject" id="1003">
+ <string key="NSClassName">FirstResponder</string>
+ </object>
+ <object class="NSCustomObject" id="1004">
+ <string key="NSClassName">NSApplication</string>
+ </object>
+ <object class="NSWindowTemplate" id="1005">
+ <int key="NSWindowStyleMask">15</int>
+ <int key="NSWindowBacking">2</int>
+ <string key="NSWindowRect">{{196, 240}, {480, 270}}</string>
+ <int key="NSWTFlags">544735232</int>
+ <string key="NSWindowTitle">Window</string>
+ <string key="NSWindowClass">NSWindow</string>
+ <nil key="NSViewClass"/>
+ <string key="NSWindowContentMaxSize">{3.40282e+38, 3.40282e+38}</string>
+ <object class="NSView" key="NSWindowView" id="1006">
+ <nil key="NSNextResponder"/>
+ <int key="NSvFlags">256</int>
+ <string key="NSFrameSize">{480, 270}</string>
+ </object>
+ <string key="NSScreenRect">{{0, 0}, {1680, 1028}}</string>
+ <string key="NSMaxSize">{3.40282e+38, 3.40282e+38}</string>
+ </object>
+ </object>
+ <object class="IBObjectContainer" key="IBDocument.Objects">
+ <object class="NSMutableArray" key="connectionRecords">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ </object>
+ <object class="IBMutableOrderedSet" key="objectRecords">
+ <object class="NSArray" key="orderedObjects">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ <object class="IBObjectRecord">
+ <int key="objectID">0</int>
+ <object class="NSArray" key="object" id="1002">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ </object>
+ <reference key="children" ref="1000"/>
+ <nil key="parent"/>
+ </object>
+ <object class="IBObjectRecord">
+ <int key="objectID">-2</int>
+ <reference key="object" ref="1001"/>
+ <reference key="parent" ref="1002"/>
+ <string type="base64-UTF8" key="objectName">RmlsZSdzIE93bmVyA</string>
+ </object>
+ <object class="IBObjectRecord">
+ <int key="objectID">-1</int>
+ <reference key="object" ref="1003"/>
+ <reference key="parent" ref="1002"/>
+ <string key="objectName">First Responder</string>
+ </object>
+ <object class="IBObjectRecord">
+ <int key="objectID">-3</int>
+ <reference key="object" ref="1004"/>
+ <reference key="parent" ref="1002"/>
+ <string key="objectName">Application</string>
+ </object>
+ <object class="IBObjectRecord">
+ <int key="objectID">1</int>
+ <reference key="object" ref="1005"/>
+ <object class="NSMutableArray" key="children">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ <reference ref="1006"/>
+ </object>
+ <reference key="parent" ref="1002"/>
+ </object>
+ <object class="IBObjectRecord">
+ <int key="objectID">2</int>
+ <reference key="object" ref="1006"/>
+ <reference key="parent" ref="1005"/>
+ </object>
+ </object>
+ </object>
+ <object class="NSMutableDictionary" key="flattenedProperties">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ <object class="NSArray" key="dict.sortedKeys">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ <string>-1.IBPluginDependency</string>
+ <string>-2.IBPluginDependency</string>
+ <string>-3.IBPluginDependency</string>
+ <string>1.IBPluginDependency</string>
+ <string>1.IBWindowTemplateEditedContentRect</string>
+ <string>1.NSWindowTemplate.visibleAtLaunch</string>
+ <string>1.WindowOrigin</string>
+ <string>1.editorWindowContentRectSynchronizationRect</string>
+ <string>2.IBPluginDependency</string>
+ </object>
+ <object class="NSMutableArray" key="dict.values">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ <string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+ <string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+ <string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+ <string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+ <string>{{357, 418}, {480, 270}}</string>
+ <integer value="1"/>
+ <string>{196, 240}</string>
+ <string>{{357, 418}, {480, 270}}</string>
+ <string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+ </object>
+ </object>
+ <object class="NSMutableDictionary" key="unlocalizedProperties">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ <object class="NSArray" key="dict.sortedKeys">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ </object>
+ <object class="NSMutableArray" key="dict.values">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ </object>
+ </object>
+ <nil key="activeLocalization"/>
+ <object class="NSMutableDictionary" key="localizations">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ <object class="NSArray" key="dict.sortedKeys">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ </object>
+ <object class="NSMutableArray" key="dict.values">
+ <bool key="EncodedWithXMLCoder">YES</bool>
+ </object>
+ </object>
+ <nil key="sourceID"/>
+ <int key="maxID">2</int>
+ </object>
+ <object class="IBClassDescriber" key="IBDocument.Classes"/>
+ <int key="IBDocument.localizationMode">0</int>
+ <bool key="IBDocument.PluginDeclaredDependenciesTrackSystemTargetVersion">YES</bool>
+ <nil key="IBDocument.LastKnownRelativeProjectPath"/>
+ <int key="IBDocument.defaultPropertyAccessControl">3</int>
+ </data>
+</archive>
diff --git a/server/djrandom/fingerprint/fingerprint.py b/server/djrandom/fingerprint/fingerprint.py
index c1fd67c859edb892bf11addfde1ef5828a57474b..a332536a47e17a854eb6ed4c4387b22d384df2f2 100644
--- a/server/djrandom/fingerprint/fingerprint.py
+++ b/server/djrandom/fingerprint/fingerprint.py
@@ -8,16 +8,19 @@ from djrandom import daemonize
from djrandom import utils
from djrandom.model.mp3 import MP3
from djrandom.database import Session, init_db
+from djrandom.model import processor
log = logging.getLogger(__name__)
-class Fingerprinter(object):
+class Fingerprinter(processor.Processor):
def __init__(self, codegen_path):
+ processor.Processor.__init__(self)
self.codegen_path = codegen_path
def process(self, mp3):
+ log.info('fingerprinting %s' % mp3.sha1)
pipe = subprocess.Popen(
[self.codegen_path, mp3.path, '10', '30'],
close_fds=False,
@@ -28,29 +31,14 @@ class Fingerprinter(object):
# (Ugly Hack!)
mp3.set_fingerprint(fp_json[2:-2])
- def compute_fingerprints(self, run_once):
- """Compute fingerprints of new files."""
- while True:
- mp3 = MP3.get_with_no_fingerprint().limit(1).first()
- if not mp3:
- if run_once:
- break
- Session.remove()
- time.sleep(300)
- continue
- log.info('fingerprinting %s' % mp3.sha1)
- try:
- self.process(mp3)
- except Exception, e:
- log.error(traceback.format_exc())
- Session.add(mp3)
- Session.commit()
+ def query(self):
+ return MP3.get_with_no_fingerprint()
def run_fingerprinter(db_url, codegen_path, run_once):
init_db(db_url)
scanner = Fingerprinter(codegen_path)
- scanner.compute_fingerprints(run_once)
+ scanner.loop(run_once=run_once)
def main():
diff --git a/server/djrandom/metadata_fixer/metadata_fixer.py b/server/djrandom/metadata_fixer/metadata_fixer.py
index a0ffcd378256fbfd930516d75c78d3177508505c..bae0d0f930cd25937ec3278c79c2c3245663e69c 100644
--- a/server/djrandom/metadata_fixer/metadata_fixer.py
+++ b/server/djrandom/metadata_fixer/metadata_fixer.py
@@ -11,6 +11,7 @@ from djrandom import utils
from djrandom.model.mp3 import MP3
from djrandom.database import Session, init_db, indexer
from djrandom.scanner import metadata
+from djrandom.model import processor
log = logging.getLogger(__name__)
@@ -19,12 +20,15 @@ class NoMetadataError(Exception):
pass
-class MetadataFixer(object):
+class MetadataFixer(processor.Processor):
ECHONEST_API_URL = 'http://developer.echonest.com/api/v4/song/identify'
- def __init__(self, echonest_api_key):
+ def __init__(self, echonest_api_key, partial, dry_run):
+ processor.Processor.__init__(self, dry_run)
self.api_key = echonest_api_key
+ self.partial = partial
+ self.n_bad = self.n_ok = self.n_err = 0
def identify_song(self, mp3):
json_fp = mp3.get_fingerprint()
@@ -56,16 +60,9 @@ class MetadataFixer(object):
log.debug('retrying...')
- def process(self, mp3):
- info = self.identify_song(mp3)
- mp3.title = metadata.normalize_string(info['title'])
- mp3.artist = metadata.normalize_string(info['artist_name'])
-
- def scan(self, dry_run, partial):
- """Scan the database for new files."""
- n_bad = n_ok = n_err = 0
- if partial:
- entries = MP3.query.filter(
+ def query(self):
+ if self.partial:
+ return MP3.query.filter(
(MP3.state == MP3.READY)
& (MP3.has_fingerprint == True)
& ((MP3.artist == None)
@@ -73,44 +70,42 @@ class MetadataFixer(object):
| (MP3.title == '')
| (MP3.title == None)))
else:
- entries = MP3.get_with_bad_metadata()
- for mp3 in entries:
- n_bad += 1
- log.info('searching metadata for %s' % mp3.sha1)
- try:
- self.process(mp3)
- mp3.state = MP3.READY
- log.info('found: %s / %s' % (mp3.artist, mp3.title))
- n_ok += 1
- except NoMetadataError:
- mp3.state = MP3.ERROR
- n_err += 1
- except Exception, e:
- log.error(traceback.format_exc())
- n_err += 1
- mp3.state = MP3.ERROR
- indexer.add_mp3(mp3)
- Session.add(mp3)
- if not dry_run:
- Session.commit()
- indexer.commit()
- log.debug('total: %d songs, found: %d' % (n_bad, n_ok))
-
- def run(self, run_once, dry_run, partial):
- while True:
- self.scan(dry_run, partial)
- if run_once:
- break
- Session.remove()
- time.sleep(600)
+ return MP3.get_with_bad_metadata()
+
+ def _process(self, mp3):
+ log.info('searching metadata for %s' % mp3.sha1)
+ info = self.identify_song(mp3)
+ mp3.title = metadata.normalize_string(info['title'])
+ mp3.artist = metadata.normalize_string(info['artist_name'])
+
+ def process(self, mp3):
+ self.n_bad += 1
+ try:
+ self._process(mp3)
+ mp3.state = MP3.READY
+ log.info('found: %s / %s' % (mp3.artist, mp3.title))
+ self.n_ok += 1
+ except NoMetadataError:
+ mp3.state = MP3.ERROR
+ self.n_err += 1
+ except Exception, e:
+ log.error(traceback.format_exc())
+ self.n_err += 1
+ mp3.state = MP3.ERROR
+ indexer.add_mp3(mp3)
+
+ def commit(self):
+ log.debug('%d songs, found: %d (%d errs)' % (
+ self.n_bad, self.n_ok, self.n_err))
+ indexer.commit()
def run_fixer(solr_url, echonest_api_key, db_url, dry_run, partial, run_once):
socket.setdefaulttimeout(300)
init_db(db_url, solr_url)
- fixer = MetadataFixer(echonest_api_key)
- fixer.run(run_once, dry_run, partial)
+ fixer = MetadataFixer(echonest_api_key, partial, dry_run)
+ fixer.loop(run_once=run_once)
def main():
diff --git a/server/djrandom/model/processor.py b/server/djrandom/model/processor.py
new file mode 100644
index 0000000000000000000000000000000000000000..8b541ae1e2a94ca282665baef16e8b9b91fc5ca5
--- /dev/null
+++ b/server/djrandom/model/processor.py
@@ -0,0 +1,60 @@
+import logging
+import traceback
+from djrandom.database import Session
+
+log = logging.getLogger(__name__)
+
+
+class Processor(object):
+ """Base class to perform operations on database objects.
+
+ Used as the basis of most cron-based jobs, it provides enough
+ abstraction to support a conversion to an event-based model if
+ necessary.
+
+ Subclasses should define at least the 'query' and 'process'
+ methods.
+ """
+
+ def __init__(self, dry_run=False):
+ self._dry_run = dry_run
+
+ def full_scan(self):
+ n = 0
+ items = self.query()
+ for item in items:
+ n += 1
+ try:
+ self.process(item)
+ Session.add(item)
+ except Exception, e:
+ log.error(
+ 'Unexpected exception processing item %s:\n %s' % (
+ item, traceback.format_exc()))
+ if n > 0 and not self._dry_run:
+ Session.commit()
+ self.commit()
+ return n
+
+ def commit(self):
+ """Optional commit hook for subclasses."""
+ pass
+
+ def query(self):
+ raise NotImplemented()
+
+ def process(self, item):
+ raise NotImplemented()
+
+ def loop(self, run_once=False, period=300):
+ while True:
+ n = 0
+ try:
+ n = self.full_scan()
+ except Exception, e:
+ log.error('Temporary error: %s' % e)
+ if run_once:
+ break
+ if not n:
+ Session.remove()
+ time.sleep(period)
diff --git a/server/djrandom/scanner/scanner.py b/server/djrandom/scanner/scanner.py
index 39b2e14515be8a23e3798ceb9ee3bf8d069344d9..021bb035fef567e4d7f6a6cb9bb0111c977308ca 100644
--- a/server/djrandom/scanner/scanner.py
+++ b/server/djrandom/scanner/scanner.py
@@ -8,6 +8,7 @@ from djrandom import utils
from djrandom.model.mp3 import MP3
from djrandom.database import Session, init_db, indexer
from djrandom.scanner import metadata
+from djrandom.model import processor
log = logging.getLogger(__name__)
@@ -16,46 +17,40 @@ class BadMetadataError(Exception):
pass
-class Scanner(object):
+class Scanner(processor.Processor):
- def process(self, mp3):
+ def _process(self, mp3):
mp3_info = metadata.analyze_mp3(mp3.path)
if not mp3_info.get('artist') or not mp3_info.get('title'):
raise BadMetadataError()
for key, value in mp3_info.iteritems():
setattr(mp3, key, value)
- def scan_db(self, run_once):
+ def query(self):
+ return MP3.query.filter_by(state=MP3.INCOMING)
+
+ def process(self, mp3):
"""Scan the database for new files."""
- while True:
- mp3 = MP3.query.filter_by(state=MP3.INCOMING
- ).limit(1).first()
- if not mp3:
- Session.remove()
- indexer.commit()
- if run_once:
- break
- time.sleep(60)
- continue
- log.info('processing %s' % mp3.sha1)
- try:
- self.process(mp3)
- mp3.state = MP3.READY
- except BadMetadataError:
- log.info('bad metadata for %s' % mp3.sha1)
- mp3.state = MP3.BAD_METADATA
- except Exception, e:
- log.error(traceback.format_exc())
- mp3.state = MP3.ERROR
- indexer.add_mp3(mp3)
- Session.add(mp3)
- Session.commit()
+ log.info('processing %s' % mp3.sha1)
+ try:
+ self._process(mp3)
+ mp3.state = MP3.READY
+ except BadMetadataError:
+ log.info('bad metadata for %s' % mp3.sha1)
+ mp3.state = MP3.BAD_METADATA
+ except Exception, e:
+ log.error(traceback.format_exc())
+ mp3.state = MP3.ERROR
+ indexer.add_mp3(mp3)
+
+ def commit(self):
+ indexer.commit()
def run_scanner(solr_url, db_url, run_once):
init_db(db_url, solr_url)
scanner = Scanner()
- scanner.scan_db(run_once)
+ scanner.loop(run_once=run_once)
def main():
diff --git a/server/djrandom/test/test_scanner.py b/server/djrandom/test/test_scanner.py
index 90cccfe8198c2fdfa5348df9079131588ee0333b..432bf85cfe24f7bdcbbc237015f0873e80e7062f 100644
--- a/server/djrandom/test/test_scanner.py
+++ b/server/djrandom/test/test_scanner.py
@@ -40,7 +40,7 @@ class ScannerTest(DbTestCase):
mp3 = MP3.query.get('1')
sc = scanner.Scanner()
- sc.process(mp3)
+ sc._process(mp3)
self.assertEquals(u'artist', mp3.artist)
self.assertEquals(u'title', mp3.title)
@@ -52,20 +52,20 @@ class ScannerTest(DbTestCase):
mp3 = MP3.query.get('1')
sc = scanner.Scanner()
self.assertRaises(scanner.BadMetadataError,
- sc.process, mp3)
+ sc._process, mp3)
def test_scanner_run(self):
sc = scanner.Scanner()
mp3 = MP3.query.get('1')
- self.mox.StubOutWithMock(sc, 'process')
- sc.process(mp3)
+ self.mox.StubOutWithMock(sc, '_process')
+ sc._process(mp3)
indexer.add_mp3(mp3)
indexer.commit()
self.mox.ReplayAll()
- sc.scan_db(run_once=True)
+ sc.loop(run_once=True)
# Verify changes to the mp3 object.
mp3b = MP3.query.get('1')
@@ -75,14 +75,14 @@ class ScannerTest(DbTestCase):
sc = scanner.Scanner()
mp3 = MP3.query.get('1')
- self.mox.StubOutWithMock(sc, 'process')
- sc.process(mp3).AndRaise(scanner.BadMetadataError())
+ self.mox.StubOutWithMock(sc, '_process')
+ sc._process(mp3).AndRaise(scanner.BadMetadataError())
indexer.add_mp3(mp3)
indexer.commit()
self.mox.ReplayAll()
- sc.scan_db(run_once=True)
+ sc.loop(run_once=True)
# Verify changes to the mp3 object.
mp3b = MP3.query.get('1')
@@ -92,14 +92,14 @@ class ScannerTest(DbTestCase):
sc = scanner.Scanner()
mp3 = MP3.query.get('1')
- self.mox.StubOutWithMock(sc, 'process')
- sc.process(mp3).AndRaise(Exception('something bad!'))
+ self.mox.StubOutWithMock(sc, '_process')
+ sc._process(mp3).AndRaise(Exception('something bad!'))
indexer.add_mp3(mp3)
indexer.commit()
self.mox.ReplayAll()
- sc.scan_db(run_once=True)
+ sc.loop(run_once=True)
# Verify changes to the mp3 object.
mp3b = MP3.query.get('1')