Merge branch 'master' of https://git.autistici.org/djrandom

client/README.txt

@@ -12,7 +12,7 @@ Install from source
 
 2. Clone the Git repository:
 
-    $ git clone https://git.autistici.org/djrandom.git
+    $ git clone http://git.autistici.org/djrandom.git
 
 3. Install the client. Python will automatically download all the
    required dependencies:
@@ -35,10 +35,24 @@ Install from package
 2. Client packages are available:
 
    OSX:
-   - https://git.autistici.org/p/djrandom/dist/DJRandomUploader.zip
+   - http://git.autistici.org/p/djrandom/dist/DJRandomUploader.zip
 
    Linux (Debian/Ubuntu):
-   - https://git.autistici.org/p/djrandom/dist/djrandom-client_0.2.3_all.deb
+   - Add the following repository to your list of APT sources (you
+     can simply create /etc/apt/sources.list.d/djrandom.list for
+     that if your apt is recent enough):
+
+       deb http://git.autistici.org/p/djrandom/debian unstable main
+
+     Install the repository GPG key with:
+
+       $ wget -O- http://git.autistici.org/p/djrandom/debian/repo.key \
+         | sudo apt-key add -
+
+     Then update and install the djrandom-client package:
+
+       $ sudo apt-get update
+       $ sudo apt-get install djrandom-client
 
 
 Running the client

client/debian/changelog

+djrandom-client (0.2.7-1) unstable; urgency=low
+
+  * Add an init script to run it as a daemon.
+
+ -- ale <ale@incal.net>  Sun, 25 Feb 2012 00:59:43 +0000
+
+djrandom-client (0.2.7) unstable; urgency=low
+
+  * Add --exclude option.
+  * Ignore metadata and hidden files.
+
+ -- ale <ale@incal.net>  Sat, 24 Feb 2012 23:26:12 +0000
+
 djrandom-client (0.2.5) unstable; urgency=low
 
   * Fix bandwidth throttling for HTTPS urls.

client/debian/control

@@ -8,7 +8,7 @@ Standards-Version: 3.8.0.1
 
 Package: djrandom-client
 Architecture: all
-Depends: ${python:Depends}, python-pyinotify
+Depends: ${python:Depends}
 Description: DJRandom uploader client.
  Uploads your music in the background.
 

client/debian/djrandom-client.default

+
+# Set this to 'true' to enable the daemon.
+ENABLED=false
+

client/debian/djrandom-client.init

+#!/bin/bash
+#
+# Start/stop the DJRandom client.
+#
+### BEGIN INIT INFO
+# Provides:          djrandom
+# Required-Start:    $network $local_fs
+# Required-Stop:
+# Should-Start:
+# Should-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: DJRandom Client
+# Description:       DJRandom Client
+### END INIT INFO
+
+
+NAME=djrandom-client
+DESCR="DJRandom client"
+DAEMON=/usr/bin/djrandom-client
+RUNDIR=/var/run/djrandom
+PIDFILE=${RUNDIR}/${NAME}.pid
+USER=nobody
+OPTIONS=
+ENABLED=false
+
+test -e /etc/default/${NAME} && . /etc/default/${NAME}
+
+test -x ${DAEMON} || exit 0
+
+if [ "${ENABLED}" != "true" ]; then
+  echo "DJRandom will not start as a daemon."
+  echo "Set ENABLED=true in /etc/default/${NAME} to enable."
+  exit 0
+fi
+
+case "$1" in
+start)
+	test -d ${RUNDIR} || mkdir -p ${RUNDIR}
+	if [ -n "${USER}" ]; then
+		chown ${USER} ${RUNDIR}
+		OPTIONS="--user ${USER} ${OPTIONS}"
+	fi
+	OPTIONS="--skip_version_check ${OPTIONS}"
+
+	echo -n "Starting ${DESCR}... "
+	start-stop-daemon --start --pidfile ${PIDFILE} \
+		   	--exec ${DAEMON} -- ${OPTIONS}
+	echo "ok"
+	;;
+stop)
+	echo -n "Stopping ${DESCR}... "
+	start-stop-daemon --stop \
+	    	--pidfile ${PIDFILE} && rm -f ${PIDFILE}
+	echo "ok"
+	;;
+restart)
+	$0 stop
+	sleep 3
+	$0 start
+	;;
+esac
+
+exit 0
+

client/djrandom_client/client.py

@@ -14,20 +14,11 @@ from djrandom_client import upload
 from djrandom_client import utils
 from djrandom_client import throttle
 
-# Detect platform and selectively enable inotify/fsevents watchers.
-watcher_support = False
-try:
-    if platform.system() == 'Darwin':
-        import djrandom_client.osx_watcher as watcher
-        watcher_support = True
-    elif platform.system() == 'Linux':
-        import djrandom_client.linux_watcher as watcher
-        watcher_support = True
-except ImportError:
-    pass
-
 log = logging.getLogger(__name__)
 
+# How often we rescan the filesystem looking for new files.
+SCAN_DELAY = 1800
+
 
 class FullScan(threading.Thread):
     """Do a recursive directory scan when starting."""
@@ -55,11 +46,10 @@ class FullScan(threading.Thread):
             time.sleep(delay)
 
 
-def run_client(server_url, music_dir, excludes, api_key, run_once, bwlimit,
-               enable_watcher):
+def run_client(server_url, music_dir, excludes, api_key, run_once, bwlimit):
     log.debug('settings: server=%s, music_dir=%s, api_key=%s, '
-              'bwlimit=%s, watcher=%s' % (
-        server_url, music_dir, api_key, bwlimit, str(enable_watcher)))
+              'bwlimit=%s' % (
+        server_url, music_dir, api_key, bwlimit))
 
     # Warn if we're running without a bandwidth limit.
     bwlimit = int(bwlimit)
@@ -82,19 +72,11 @@ def run_client(server_url, music_dir, excludes, api_key, run_once, bwlimit,
             exclude_prefix += '/'
         abs_excludes.append(os.path.realpath(exclude_prefix))
 
-    if enable_watcher and not watcher_support:
-        log.warn('inotify/fsevents support not enabled on this platform')
-        enable_watcher = False
-
     upl = upload.Uploader(server_url.rstrip('/'), api_key, abs_excludes)
     upl.setDaemon(True)
 
     # Start the full filesystem scan in the background.
-    if enable_watcher:
-        scan_delay = 3 * 86400
-    else:
-        scan_delay = 9600
-    scan = FullScan(music_dir, upl.queue, scan_delay, run_once)
+    scan = FullScan(music_dir, upl.queue, SCAN_DELAY, run_once)
     scan.setDaemon(True)
 
     if not run_once:
@@ -104,15 +86,8 @@ def run_client(server_url, music_dir, excludes, api_key, run_once, bwlimit,
             # Set 'idle' I/O scheduling class, we won't disturb other programs.
             os.system('/usr/bin/ionice -c 3 -p %d' % os.getpid())
 
-        # Start the live filesystem watcher.
-        if enable_watcher:
-            wtch = watcher.Watcher(music_dir, upl.queue)
-
         # Install termination signal handlers.
         def _cleanup(signum, frame):
-            if enable_watcher:
-                log.info('stopping watcher...')
-                wtch.stop()
             log.info('got signal %d, exiting...' % signum)
             upl.stop()
             sys.exit(0)
@@ -152,8 +127,6 @@ def main():
     parser.add_option('--bwlimit', type='int', default=0,
                       help='Upload bandwidth limit, kilobytes/s (default: '
                       'unlimited)')
-    parser.add_option('--no_realtime_watch', action='store_true',
-                      help='Monitor music_dir in realtime')
     parser.add_option('--skip_version_check', action='store_true')
     daemonize.add_standard_options(parser)
     utils.read_config_defaults(
@@ -171,13 +144,9 @@ def main():
     if not opts.skip_version_check and utils.check_version():
         print >>sys.stderr, 'A new release is available! Please update.'
 
-    # Reading from the configuration file will set this variable to
-    # a string, convert it back into boolean.
-    do_realtime = not opts.no_realtime_watch
-
     daemonize.daemonize(opts, run_client, 
                         (opts.server_url, opts.music_dir, opts.exclude, 
-                         opts.api_key, opts.once, opts.bwlimit, do_realtime))
+                         opts.api_key, opts.once, opts.bwlimit))
 
 
 if __name__ == '__main__':

client/djrandom_client/filescan.py

 import os
 
+DIR_BLACKLIST = set(['.AppleDouble'])
+
 
 def recursive_scan(basedir, queue):
     n = 0
-    for root, dirs, files in os.walk(basedir):
+    for root, dirs, files in os.walk(basedir, topdown=True,
+                                     followlinks=True):
+        prune_dirs = [x for x in dirs
+                      if (x in DIR_BLACKLIST or x.startswith('.'))]
+        for pdir in prune_dirs:
+            dirs.remove(pdir)
         for filename in files:
-            if filename.lower().endswith('.mp3'):
+            if (filename.lower().endswith('.mp3') and
+                not filename.startswith('.')):
                 path = os.path.join(root, filename)
                 queue.put(path)
                 n += 1

client/djrandom_client/linux_watcher.py

-import logging
-import pyinotify
-
-log = logging.getLogger(__name__)
-
-
-class Watcher(pyinotify.ProcessEvent):
-
-    def __init__(self, base, queue):
-        self.queue = queue
-        self.wm = pyinotify.WatchManager()
-        self.notifier = pyinotify.ThreadedNotifier(self.wm, self)
-        self.notifier.setDaemon(True)
-        self.notifier.start()
-        self.wm.add_watch(base, pyinotify.IN_CLOSE_WRITE, rec=True)
-
-    def stop(self):
-        self.notifier.stop()
-
-    def process_IN_CLOSE(self, event):
-        if event.pathname.lower().endswith('.mp3'):
-            log.debug('event in %s: %x' % (event.pathname, event.mask))
-            self.queue.put(event.pathname)
-

client/djrandom_client/osx_watcher.py

-import logging
-from fsevents import Observer, Stream
-from djrandom_client import filescan
-
-log = logging.getLogger(__name__)
-
-
-class Watcher(object):
-
-    def __init__(self, base, queue):
-        self.queue = queue
-        self.observer = Observer()
-        self.observer.schedule(Stream(self.callback, base))
-        self.observer.setDaemon(True)
-        self.observer.start()
-
-    def callback(self, subpath, mask):
-        log.debug('event in %s: %x' % (subpath, mask))
-        filescan.directory_scan(subpath, self.queue)
-
-    def stop(self):
-        self.observer.stop()
-

client/djrandom_client/test/test_client.py

@@ -161,12 +161,12 @@ class ClientOptionsTest(ClientRunner, mox.MoxTestBase):
                                          '/my/music',
                                          [],
                                          'KEY',
-                                         True, 10, False))
+                                         True, 10))
 
         self.mox.ReplayAll()
         self._Run(['--api_key=KEY', '--server_url=http://server/receiver',
                    '--music_dir=/my/music', '--bwlimit=10',
-                   '--once', '--no_realtime_watch'])
+                   '--once'])
 
 
 class RunClientTest(mox.MoxTestBase):
@@ -201,8 +201,7 @@ class RunClientTest(mox.MoxTestBase):
             [],
             'KEY',
             True,
-            150,
-            False)
+            150)
 
 
 if __name__ == '__main__':

client/djrandom_client/upload.py

@@ -113,6 +113,7 @@ class Uploader(threading.Thread):
         if not result:
             raise UploadError('server error')
         self.stats.incr('uploaded_files')
+        self.stats.incr('uploaded_bytes', os.path.getsize(path))
         log.info('successfully uploaded %s (%s)' % (path, sha1))
 
     def _should_exclude(self, path):

client/djrandom_client/version.py

-VERSION = '0.2.6'
+VERSION = '0.2.7'

client/osx-ui/DJRandomUploader/ProcessController.m

@@ -54,7 +54,6 @@
         NSArray *args = [NSArray 
                          arrayWithObjects:@"--api_key", curApiKey, @"--music_dir", 
                          curMusicFolder, @"--debug", @"--foreground", 
-                         @"--no_realtime_watch",
                          nil];
         if ([curBwLimit intValue] > 0) {
             args = [args arrayByAddingObjectsFromArray:[NSArray 

client/setup.py

@@ -8,13 +8,11 @@ from djrandom_client import version
 from setuptools import setup, find_packages
 import platform
 if platform.system() == 'Darwin':
-    platform_requires = ['MacFSEvents']
     extra_options = {
       'app': ['djrandom_client/client.py'],
       'setup_requires': ['py2app']
     }
 else:
-    platform_requires = ['pyinotify']
     extra_options = {}
 
 setup(
@@ -24,7 +22,6 @@ setup(
   author="ale",
   author_email="ale@incal.net",
   url="http://git.autistici.org/git/djrandom.git",
-  install_requires=platform_requires,
   zip_safe=True,
   packages=find_packages(),
   entry_points={