feature extraction using the Marsyas toolkit and our own SWIG-wrapped tiny API

server/djrandom/model/mp3.py

@@ -18,6 +18,14 @@ class Fingerprint(Base):
     echoprint_fp = Column(Text())
 
 
+class Features(Base):
+
+    __tablename__ = 'features'
+
+    sha1 = Column(String(40), primary_key=True)
+    timbre_vector = Column(Text())
+
+
 class MP3(Base):
     """A single MP3.
 
@@ -51,6 +59,12 @@ class MP3(Base):
                                 foreign_keys=Fingerprint.sha1,
                                 uselist=False)
 
+    has_features = Column(Boolean, default=False)
+    features = relationship(Features,
+                            primaryjoin=sha1 == Features.sha1,
+                            foreign_keys=Features.sha1,
+                            uselist=False)
+
     def __init__(self, **kw):
         for k, v in kw.items():
             setattr(self, k, v)
@@ -114,12 +128,32 @@ class MP3(Base):
         self.has_fingerprint = True
         Session.add(fpobj)
 
+    def get_features(self):
+        if self.has_features:
+            return self.features
+
+    def set_features(self, **args):
+        fobj = self.features
+        if not fobj:
+            fobj = Features()
+            fobj.sha1 = self.sha1
+            self.features = fobj
+        for k, v in args.iteritems():
+            setattr(fobj, k, v)
+        self.has_features = True
+        Session.add(fobj)
+
     @classmethod
     def get_with_no_fingerprint(cls):
         return cls.query.filter(((cls.state == cls.READY)
                                  | (cls.state == cls.BAD_METADATA))
                                 & (cls.has_fingerprint == 0))
 
+    @classmethod
+    def get_with_no_features(cls):
+        return cls.query.filter((cls.state == cls.READY)
+                                & (cls.has_fingerprint == 0))
+
     @classmethod
     def get_with_bad_metadata(cls):
         return cls.query.filter_by(state=cls.BAD_METADATA,

server/djrandom/mood/Makefile

+
+all: mood.py
+
+mood_wrap.cc mood.py: mood.i
+	swig -o mood_wrap.cc -c++ -python mood.i

server/djrandom/mood/extract_features.py

+import os
+import optparse
+import logging
+import subprocess
+import time
+import traceback
+from djrandom import daemonize
+from djrandom import utils
+from djrandom.model.mp3 import MP3
+from djrandom.database import Session, init_db
+from djrandom.model import processor
+from djrandom.mood import mood
+
+log = logging.getLogger(__name__)
+
+
+class FeatureExtractor(processor.Processor):
+
+    def process(self, mp3):
+        log.info('extracting features from %s' % mp3.sha1)
+        try:
+            timbre_vector = mood.vector_from_file(mp3.path)
+        except Exception, e:
+            log.error('error processing %s: %s' % (mp3.sha1, e))
+            return
+        mp3.set_features(timbre_vector=timbre_vector)
+
+    def query(self):
+        return MP3.get_with_no_features()
+
+
+def run_feature_extractor(db_url, run_once):
+    init_db(db_url)
+    scanner = FeatureExtractor()
+    scanner.loop(run_once=run_once)
+
+
+def main():
+    parser = optparse.OptionParser()
+    parser.add_option('--once', action='store_true')
+    parser.add_option('--db_url')
+    daemonize.add_standard_options(parser)
+    utils.read_config_defaults(
+        parser, os.getenv('DJRANDOM_CONF', '/etc/djrandom.conf'))
+    opts, args = parser.parse_args()
+    if not opts.db_url:
+        parser.error('Must provide --db_url')
+    if args:
+        parser.error('Too many arguments')
+
+    if opts.once:
+        opts.foreground = True
+
+    daemonize.daemonize(opts, run_feature_extractor,
+                        (opts.db_url, opts.once))
+
+
+if __name__ == '__main__':
+    main()

server/djrandom/mood/mood.cc

+
+#include <algorithm>
+#include <marsyas/MarSystemManager.h>
+#include <marsyas/NumericLib.h>
+#include "mood.h"
+
+using namespace std;
+using namespace Marsyas;
+
+namespace mood {
+
+typedef pair<string, double> distpair;
+
+struct CompareDistances {
+   bool operator()(const distpair& lhs, const distpair& rhs) {
+     return lhs.second < rhs.second;
+   }
+};
+
+vector<string>
+Mood::compare(const string& ref_key, const string& ref_vector_str, int n)
+{
+  vector<string> result;
+  vector<distpair> dists;
+  realvec ref_vector = vector_from_string(ref_vector_str);
+  realvec cov;
+
+  vector_map_t::const_iterator i;
+  for (i = vectors_.begin(); i != vectors_.end(); i++) {
+    string key = (*i).first;
+    realvec sig = (*i).second;
+
+    // Skip comparing against the reference key.
+    if (key == ref_key)
+       continue;
+
+    // Compute signature and distance.
+    double distance = NumericLib::euclideanDistance(ref_vector, sig, cov);
+    dists.push_back(make_pair(key, distance));
+  }
+
+  n = min(n, (int)dists.size());
+  partial_sort(dists.begin(), dists.begin() + n, dists.end(),
+               CompareDistances());
+
+  // Extract a vector containing just the keys.
+  vector<distpair>::const_iterator j;
+  for (j = dists.begin(); j != dists.end(); j++) {
+    result.push_back((*j).first);
+  }
+  return result;
+}
+
+
+string vector_from_file(const string& filename)
+{
+  MarSystemManager mng;
+  
+  //------------------ Feature Network ------------------------------
+
+  // Decode the file, downmix to mono, and downsample
+  MarSystem *fnet = mng.create("Series", "fnet");
+  fnet->addMarSystem(mng.create("SoundFileSource", "src"));
+  fnet->addMarSystem(mng.create("DownSampler", "ds"));
+  fnet->addMarSystem(mng.create("Stereo2Mono", "s2m"));
+  
+  // Create the feature extractor
+  fnet->addMarSystem(mng.create("TimbreFeatures", "tf"));
+  fnet->updctrl("TimbreFeatures/tf/mrs_string/enableTDChild", "ZeroCrossings/zcrs");
+  fnet->updctrl("TimbreFeatures/tf/mrs_string/enableSPChild", "MFCC/mfcc");
+  fnet->updctrl("TimbreFeatures/tf/mrs_string/enableSPChild", "Centroid/cntrd");
+  fnet->updctrl("TimbreFeatures/tf/mrs_string/enableSPChild", "Flux/flux");
+  fnet->updctrl("TimbreFeatures/tf/mrs_string/enableSPChild", "Rolloff/rlf");
+  
+  // Add the texture statistics
+  fnet->addMarSystem(mng.create("TextureStats", "tStats"));
+
+  //------------------- Set Parameters ------------------------------
+  
+  // Set the texture memory size to a 1-second window (22 analysis frames)
+  fnet->updctrl("TextureStats/tStats/mrs_natural/memSize", 22);
+
+  // Set the file name
+  fnet->updctrl("SoundFileSource/src/mrs_string/filename", filename);
+  
+  // Set the sample rate to 11250 Hz
+  mrs_natural factor = round(fnet->getctrl("SoundFileSource/src/mrs_real/osrate")
+                             ->to<mrs_real>()/11250.0);
+  fnet->updctrl("DownSampler/ds/mrs_natural/factor", factor);
+  
+  // Set the window to 1024 samples at 11250 Hz
+  // Should be able to set with simply TimbreFeatures/tf/mrs_natural/winSize,
+  // but that doesn't seem to work
+  fnet->updctrl("TimbreFeatures/tf/Series/timeDomain/ShiftInput/si/mrs_natural/winSize", 1024);
+  fnet->updctrl("TimbreFeatures/tf/Series/spectralShape/ShiftInput/si/mrs_natural/winSize", 1024);
+  fnet->updctrl("TimbreFeatures/tf/Series/lpcFeatures/ShiftInput/si/mrs_natural/winSize", 1024);
+  
+  // Find the length of the song
+  mrs_natural slength = fnet->getctrl("SoundFileSource/src/mrs_natural/size")->to<mrs_natural>();
+  
+  // Find the number of samples resulting in a whole number of analysis windows by truncating
+  mrs_natural numsamps = (mrs_natural)(((30*11250.0*factor)/512)*512);
+
+  // Shift the start over so that the duration is in the middle
+  mrs_natural start = (slength - numsamps)/2;
+
+  fnet->updctrl("SoundFileSource/src/mrs_natural/start", start);
+  fnet->updctrl("SoundFileSource/src/mrs_natural/duration", numsamps);
+
+  // ----------------- Accumulator ---------------------------------
+
+  // Accumulate over the entire song
+  MarSystem *acc = mng.create("Accumulator", "acc");
+    
+  // nTimes is measured in number of analysis windows
+  acc->updctrl("mrs_natural/nTimes", (mrs_natural)((30*11250.0)/512));
+
+  //------------------ Song Statistics -----------------------------
+  // Fanout and calculate mean and standard deviation
+  MarSystem *sstats = mng.create("Fanout", "sstats");
+  sstats->addMarSystem(mng.create("Mean", "smn"));
+  sstats->addMarSystem(mng.create("StandardDeviation", "sstd"));
+
+  // ----------------- Top Level Network Wrapper -------------------
+  
+  // (src->downmix->downsample->features->texture stats)
+  // --->accumulator->song stats->output
+  MarSystem *tnet = mng.create("Series", "tnet");
+  acc->addMarSystem(fnet);
+  tnet->addMarSystem(acc);
+  tnet->addMarSystem(sstats);
+  
+  // set the hop size to 512 (needs to be set for the top-level network)
+  tnet->updctrl("mrs_natural/inSamples", factor*512);
+
+  // Should only need to tick once
+  tnet->tick();
+
+  return vector_to_string(
+    tnet->getctrl("mrs_realvec/processedData")->to<mrs_realvec>());
+}
+
+
+string vector_to_string(const realvec& v)
+{
+  ostringstream ostr;
+  ostr << v;
+  return ostr.str();
+}
+
+
+realvec vector_from_string(const string& vstr)
+{
+  istringstream istr(vstr);
+  realvec result;
+  istr >> result;
+  return result;
+}
+
+} // namespace mood

server/djrandom/mood/mood.h

+// -*- c++ -*-
+
+#ifndef __djrandom_mood_H
+#define __djrandom_mood_H 1
+
+#include <map>
+#include <vector>
+#include <string>
+#include <marsyas/realvec.h>
+
+namespace mood {
+
+typedef std::map<std::string, Marsyas::realvec> vector_map_t;
+
+class Mood
+{
+public:
+  Mood(const vector_map_t& vectors)
+    : vectors_(vectors)
+  {}
+
+  virtual ~Mood()
+  {}
+
+  std::vector<std::string> compare(const std::string& ref_key,
+                                   const std::string& ref_vector_str,
+                                   int n);
+
+protected:
+  vector_map_t vectors_;
+};
+
+// Extract a (serialized) vector from an MP3 file.
+std::string vector_from_file(const std::string& filename);
+
+// Internal serialization functions.
+std::string vector_to_string(const Marsyas::realvec& v);
+Marsyas::realvec vector_from_string(const std::string& vstr);
+
+
+} // namespace mood
+
+#endif

server/djrandom/mood/mood.i

+%module mood
+%include "std_vector.i"
+%include "std_string.i"
+
+%{
+#define SWIG_FILE_WITH_INIT
+#include "mood.h"
+%}
+
+%typemap(in) (const std::map<std::string, Marsyas::realvec>& vectors) {
+
+  // Check type.
+  if (!PyDict_Check($input)) {
+    PyErr_SetString(PyExc_TypeError, "Not a dict");
+    return NULL;
+  }
+
+  $1 = new mood::vector_map_t;
+
+  // Iterate on the Python dictionary and build the unserialized
+  // representation.
+  PyObject *key, *value;
+  Py_ssize_t pos = 0;
+  while (PyDict_Next($input, &pos, &key, &value)) {
+    $1->insert(
+      std::make_pair(
+        PyString_AsString(key),
+        mood::vector_from_string(PyString_AsString(value))));
+  }
+
+}
+
+%include "mood.h"
+

server/djrandom/mood/mood.py

+# This file was automatically generated by SWIG (http://www.swig.org).
+# Version 1.3.31
+#
+# Don't modify this file, modify the SWIG interface instead.
+# This file is compatible with both classic and new-style classes.
+
+import _mood
+import new
+new_instancemethod = new.instancemethod
+try:
+    _swig_property = property
+except NameError:
+    pass # Python < 2.2 doesn't have 'property'.
+def _swig_setattr_nondynamic(self,class_type,name,value,static=1):
+    if (name == "thisown"): return self.this.own(value)
+    if (name == "this"):
+        if type(value).__name__ == 'PySwigObject':
+            self.__dict__[name] = value
+            return
+    method = class_type.__swig_setmethods__.get(name,None)
+    if method: return method(self,value)
+    if (not static) or hasattr(self,name):
+        self.__dict__[name] = value
+    else:
+        raise AttributeError("You cannot add attributes to %s" % self)
+
+def _swig_setattr(self,class_type,name,value):
+    return _swig_setattr_nondynamic(self,class_type,name,value,0)
+
+def _swig_getattr(self,class_type,name):
+    if (name == "thisown"): return self.this.own()
+    method = class_type.__swig_getmethods__.get(name,None)
+    if method: return method(self)
+    raise AttributeError,name
+
+def _swig_repr(self):
+    try: strthis = "proxy of " + self.this.__repr__()
+    except: strthis = ""
+    return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,)
+
+import types
+try:
+    _object = types.ObjectType
+    _newclass = 1
+except AttributeError:
+    class _object : pass
+    _newclass = 0
+del types
+
+
+class PySwigIterator(_object):
+    __swig_setmethods__ = {}
+    __setattr__ = lambda self, name, value: _swig_setattr(self, PySwigIterator, name, value)
+    __swig_getmethods__ = {}
+    __getattr__ = lambda self, name: _swig_getattr(self, PySwigIterator, name)
+    def __init__(self): raise AttributeError, "No constructor defined"
+    __repr__ = _swig_repr
+    __swig_destroy__ = _mood.delete_PySwigIterator
+    __del__ = lambda self : None;
+    def value(*args): return _mood.PySwigIterator_value(*args)
+    def incr(*args): return _mood.PySwigIterator_incr(*args)
+    def decr(*args): return _mood.PySwigIterator_decr(*args)
+    def distance(*args): return _mood.PySwigIterator_distance(*args)
+    def equal(*args): return _mood.PySwigIterator_equal(*args)
+    def copy(*args): return _mood.PySwigIterator_copy(*args)
+    def next(*args): return _mood.PySwigIterator_next(*args)
+    def previous(*args): return _mood.PySwigIterator_previous(*args)
+    def advance(*args): return _mood.PySwigIterator_advance(*args)
+    def __eq__(*args): return _mood.PySwigIterator___eq__(*args)
+    def __ne__(*args): return _mood.PySwigIterator___ne__(*args)
+    def __iadd__(*args): return _mood.PySwigIterator___iadd__(*args)
+    def __isub__(*args): return _mood.PySwigIterator___isub__(*args)
+    def __add__(*args): return _mood.PySwigIterator___add__(*args)
+    def __sub__(*args): return _mood.PySwigIterator___sub__(*args)
+    def __iter__(self): return self
+PySwigIterator_swigregister = _mood.PySwigIterator_swigregister
+PySwigIterator_swigregister(PySwigIterator)
+
+__djrandom_mood_H = _mood.__djrandom_mood_H
+class Mood(_object):
+    __swig_setmethods__ = {}
+    __setattr__ = lambda self, name, value: _swig_setattr(self, Mood, name, value)
+    __swig_getmethods__ = {}
+    __getattr__ = lambda self, name: _swig_getattr(self, Mood, name)
+    __repr__ = _swig_repr
+    def __init__(self, *args): 
+        this = _mood.new_Mood(*args)
+        try: self.this.append(this)
+        except: self.this = this
+    __swig_destroy__ = _mood.delete_Mood
+    __del__ = lambda self : None;
+    def compare(*args): return _mood.Mood_compare(*args)
+Mood_swigregister = _mood.Mood_swigregister
+Mood_swigregister(Mood)
+
+vector_from_file = _mood.vector_from_file
+vector_to_string = _mood.vector_to_string
+vector_from_string = _mood.vector_from_string
+
+

server/djrandom/mood/setup.py

+from distutils.core import setup, Extension
+
+mood_module = Extension(
+    '_mood',
+    sources=['mood_wrap.cc', 'mood.cc'],
+    libraries=['marsyas'],
+    )
+
+setup(name='mood',
+      version='0.1',
+      author='ale@incal.net',
+      description='suchen',
+      ext_modules=[mood_module],
+      py_modules=['mood'],
+      )

server/setup.py

 #!/usr/bin/python
 
-from setuptools import setup, find_packages
+from setuptools import setup, find_packages, Extension
+
+mood_module = Extension(
+    'djrandom.mood._mood',
+    sources=['djrandom/mood/mood_wrap.cc',
+             'djrandom/mood/mood.cc'],
+    libraries=['marsyas'],
+    )
 
 setup(
   name="djrandom",
@@ -13,6 +20,7 @@ setup(
   setup_requires=[],
   zip_safe=False,
   packages=find_packages(),
+  ext_modules=[mood_module],
   entry_points={
     "console_scripts": [
       "djrandom-receiver = djrandom.receiver.receiver:main",
@@ -24,6 +32,7 @@ setup(
       "djrandom-update-markov = djrandom.model.markov:main",
       "djrandom-metadata-fixer = djrandom.metadata_fixer.metadata_fixer:main",
       "djrandom-solr-fixer = djrandom.model.verify:main",
+      "djrandom-extract-features = djrandom.mood.extract_features:main",
     ],
   },
   )