diff --git a/server/djrandom/model/external.py b/server/djrandom/model/external.py
index 2708dc6b7a46000dbf119f642c6989aa583407d0..c09c4cf73973ee998d098abda865e5dc7233d4a1 100644
--- a/server/djrandom/model/external.py
+++ b/server/djrandom/model/external.py
@@ -82,9 +82,8 @@ class AlbumImageRetriever(object):
                     img = xp(xml)
                     if img:
                         self.cache.download(artist, album, img[0].text)
-                    else:
-                        self.cache.set_negative_match(artist, album)
                 except:
+                    self.cache.set_negative_match(artist, album)
                     continue
                 break
         return self.cache.get(artist, album)