Modified database storage to fallback to file storage. Added management

commands to handle bulk load and dump of files to/from the filesystem.

.gitignore

 *.pyc
 /build
+.project
+.pydevproject
+.settings
\ No newline at end of file

database_files/management/commands/database_files_dump.py

+import os
+
+from django.conf import settings
+from django.core.files.storage import default_storage
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import FileField, ImageField
+
+from database_files.models import File
+
+from optparse import make_option
+
+class Command(BaseCommand):
+    option_list = BaseCommand.option_list + (
+        make_option('-w', '--overwrite', action='store_true',
+            dest='overwrite', default=False,
+            help='If given, overwrites any existing files.'),
+    )
+    help = 'Dumps all files in the database referenced by FileFields ' + \
+        'or ImageFields onto the filesystem in the directory specified by ' + \
+        'MEDIA_ROOT.'
+
+    def handle(self, *args, **options):
+        tmp_debug = settings.DEBUG
+        settings.DEBUG = False
+        try:
+            q = File.objects.all()
+            total = q.count()
+            i = 0
+            for file in q:
+                i += 1
+                print '%i of %i' % (i, total)
+                fqfn = os.path.join(settings.MEDIA_ROOT, file.name)
+                fqfn = os.path.normpath(fqfn)
+                if os.path.isfile(fqfn) and not options['overwrite']:
+                    continue
+                dirs,fn = os.path.split(fqfn)
+                if not os.path.isdir(dirs):
+                    os.makedirs(dirs)
+                open(fqfn, 'wb').write(file.content)
+        finally:
+            settings.DEBUG = tmp_debug
+            
\ No newline at end of file

database_files/management/commands/database_files_load.py

+import os
+
+from django.conf import settings
+from django.core.files.storage import default_storage
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import FileField, ImageField
+
+from optparse import make_option
+
+class Command(BaseCommand):
+    args = ''
+    help = 'Loads all files on the filesystem referenced by FileFields ' + \
+        'or ImageFields into the database. This should only need to be ' + \
+        'done once, when initially migrating a legacy system.'
+
+    def handle(self, *args, **options):
+        tmp_debug = settings.DEBUG
+        settings.DEBUG = False
+        try:
+            broken = 0 # Number of db records referencing missing files.
+            from django.db.models import get_models
+            for model in get_models():
+               for field in model._meta.fields:
+                    if not isinstance(field, (FileField, ImageField)):
+                        continue
+                    print model.__name__, field.name
+                    # Ignore records with null or empty string values.
+                    q = {'%s__isnull'%field.name:False}
+                    xq = {field.name:''}
+                    for row in model.objects.filter(**q).exclude(**xq):
+                        try:
+                            file = getattr(row, field.name)
+                            if file is None:
+                                continue
+                            if not file.name:
+                                continue
+                            if file.path and not os.path.isfile(file.path):
+                                broken += 1
+                                continue
+                            file.read()
+                            row.save()
+                        except IOError:
+                            broken += 1
+            print '-'*80
+            print '%i broken' % (broken,)
+        finally:
+            settings.DEBUG = tmp_debug

database_files/manager.py

@@ -3,4 +3,6 @@ import os
 
 class FileManager(models.Manager):
     def get_from_name(self, name):
-        return self.get(pk=os.path.splitext(os.path.split(name)[1])[0])
+#        print 'name:',name
+#        return self.get(pk=os.path.splitext(os.path.split(name)[1])[0])
+        return self.get(name=name)
\ No newline at end of file

database_files/migrations/0001_initial.py

-
+# encoding: utf-8
+import datetime
 from south.db import db
+from south.v2 import SchemaMigration
 from django.db import models
-from database_files.models import *
 
-class Migration:
-    
+class Migration(SchemaMigration):
+
     def forwards(self, orm):
         
         # Adding model 'File'
         db.create_table('database_files_file', (
-            ('id', orm['database_files.File:id']),
-            ('content', orm['database_files.File:content']),
-            ('size', orm['database_files.File:size']),
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255, db_index=True)),
+            ('size', self.gf('django.db.models.fields.PositiveIntegerField')()),
+            ('_content', self.gf('django.db.models.fields.TextField')(db_column='content')),
         ))
         db.send_create_signal('database_files', ['File'])
-        
-    
-    
+
+
     def backwards(self, orm):
         
         # Deleting model 'File'
         db.delete_table('database_files_file')
-        
-    
-    
+
+
     models = {
         'database_files.file': {
-            'content': ('django.db.models.fields.TextField', [], {}),
+            'Meta': {'object_name': 'File'},
+            '_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}),
             'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'size': ('django.db.models.fields.IntegerField', [], {})
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}),
+            'size': ('django.db.models.fields.PositiveIntegerField', [], {})
         }
     }
-    
+
     complete_apps = ['database_files']

database_files/models.py

+import base64
+
 from django.db import models
+
 from database_files.manager import FileManager
 
 class File(models.Model):
-    content = models.TextField()
-    size = models.IntegerField()
     
     objects = FileManager()
+    
+    name = models.CharField(
+        max_length=255,
+        unique=True,
+        blank=False,
+        null=False,
+        db_index=True)
+    
+    size = models.PositiveIntegerField(
+        blank=False,
+        null=False)
 
+    _content = models.TextField(db_column='content')
+    
+    @property
+    def content(self):
+        return base64.b64decode(self._content)
+    
+    @content.setter
+    def content(self, v):
+        self._content = base64.b64encode(v)
+        
\ No newline at end of file

database_files/storage.py

-import base64
-from database_files import models
-from django.core import files
-from django.core.files.storage import Storage
-from django.core.urlresolvers import reverse
 import os
 import StringIO
 
-class DatabaseStorage(Storage):
+from django.conf import settings
+from django.core import files
+from django.core.files.storage import FileSystemStorage
+from django.core.urlresolvers import reverse
+
+from database_files import models
+
+class DatabaseStorage(FileSystemStorage):
+    
     def _generate_name(self, name, pk):
         """
         Replaces the filename with the specified pk and removes any dir
         """
-        dir_name, file_name = os.path.split(name)
-        file_root, file_ext = os.path.splitext(file_name)
-        return '%s%s' % (pk, file_ext)
+        #dir_name, file_name = os.path.split(name)
+        #file_root, file_ext = os.path.splitext(file_name)
+        #return '%s%s' % (pk, file_name)
+        return name
     
     def _open(self, name, mode='rb'):
+        """
+        Open file with filename `name` from the database.
+        """
         try:
+            # Load file from database.
             f = models.File.objects.get_from_name(name)
+            content = f.content
+            size = f.size
         except models.File.DoesNotExist:
-            return None
-        fh = StringIO.StringIO(base64.b64decode(f.content))
+            # If not yet in the database, check the local file system
+            # and load it into the database if present.
+            fqfn = self.path(name)
+            if os.path.isfile(fqfn):
+                self._save(name, open(fqfn, mode))
+                fh = super(DatabaseStorage, self)._open(name, mode)
+                content = fh.read()
+                size = fh.size
+            else:
+                # Otherwise we don't know where the file is.
+                return
+        # Normalize the content to a new file object.
+        fh = StringIO.StringIO(content)
         fh.name = name
         fh.mode = mode
-        fh.size = f.size
-        return files.File(fh)
+        fh.size = size
+        o = files.File(fh)
+        return o
     
     def _save(self, name, content):
+        """
+        Save file with filename `name` and given content to the database.
+        """
+        full_path = self.path(name)
+        try:
+            size = content.size
+        except AttributeError:
+            size = os.path.getsize(full_path)
         f = models.File.objects.create(
-            content=base64.b64encode(content.read()),
-            size=content.size,
+            content=content.read(),
+            size=size,
+            name=name,
         )
         return self._generate_name(name, f.pk)
     
     def exists(self, name):
         """
-        We generate a new filename for each file, so it will never already 
-        exist.
+        Returns true if a file with the given filename exists in the database.
+        Returns false otherwise.
         """
-        return False
+        if models.File.objects.filter(name=name).count() > 0:
+            return True
+        return super(DatabaseStorage, self).exists(name)
     
     def delete(self, name):
+        """
+        Deletes the file with filename `name` from the database and filesystem.
+        """
+        full_path = self.path(name)
         try:
             models.File.objects.get_from_name(name).delete()
         except models.File.DoesNotExist:
             pass
+        return super(DatabaseStorage, self).delete(name)
     
     def url(self, name):
-        return reverse('database_file', kwargs={'name': name})
+        """
+        Returns the web-accessible URL for the file with filename `name`.
+        """
+        return os.path.join(settings.MEDIA_URL, name)
+        #return reverse('database_file', kwargs={'name': name})
     
     def size(self, name):
+        """
+        Returns the size of the file with filename `name` in bytes.
+        """
+        full_path = self.path(name)
         try:
             return models.File.objects.get_from_name(name).size
         except models.File.DoesNotExist:
-            return 0
-
+            return super(DatabaseStorage, self).size(name)

database_files/tests/fixtures/test_data.json

 [
     {
-        "pk": 1, 
-        "model": "database_files.file", 
+        "pk": 1,
+        "model": "database_files.file",
         "fields": {
-            "content": "MTIzNDU2Nzg5MA==",
+            "name": "1.txt",
+            "_content": "MTIzNDU2Nzg5MA==",
             "size": 10
         }
     }
-]
+]
\ No newline at end of file

database_files/tests/models.py

 from django.db import models
 
 class Thing(models.Model):
-    upload = models.FileField(upload_to='not required')
-
+    upload = models.FileField(upload_to='i/special')

database_files/tests/settings.py

-DATABASE_ENGINE = 'sqlite3'
+import os, sys
+PROJECT_DIR = os.path.dirname(__file__)
+DATABASES = {
+    'default':{
+        'ENGINE': 'django.db.backends.sqlite3',
+        # Don't do this. It dramatically slows down the test.
+#        'NAME': '/tmp/database_files.db',
+#        'TEST_NAME': '/tmp/database_files.db',
+    }
+}
 ROOT_URLCONF = 'database_files.urls'
 INSTALLED_APPS = [
     'django.contrib.auth',
@@ -9,3 +18,4 @@ INSTALLED_APPS = [
     'database_files.tests',
 ]
 DEFAULT_FILE_STORAGE = 'database_files.storage.DatabaseStorage'
+MEDIA_ROOT = os.path.join(PROJECT_DIR, 'media')
\ No newline at end of file

database_files/tests/tests.py

+import os
+import StringIO
+
 from django.core import files
 from django.test import TestCase
+from django.core.files.storage import default_storage
+
 from database_files.models import File
 from database_files.tests.models import Thing
-import StringIO
+
+DIR = os.path.abspath(os.path.split(__file__)[0])
 
 class DatabaseFilesTestCase(TestCase):
     def test_adding_file(self):
+        
+        # Create default thing storing reference to file
+        # in the local media directory.
+        fqfn = os.path.join(DIR,'media/i/special/test.txt')
+        open(fqfn,'w').write('hello there')
+        o = Thing()
+        o.upload = 'i/special/test.txt'
+        o.save()
+        id = o.id
+        
+        # Confirm thing was saved.
+        Thing.objects.update()
+        q = Thing.objects.all()
+        self.assertEqual(q.count(), 1)
+        self.assertEqual(q[0].upload.name, 'i/special/test.txt')
+        
+        # Confirm the file only exists on the file system
+        # and hasn't been loaded into the database.
+        q = File.objects.all()
+        self.assertEqual(q.count(), 0)
+        
+        # Verify we can read the contents of thing.
+        o = Thing.objects.get(id=id)
+        self.assertEqual(o.upload.read(), "hello there")
+        
+        # Verify that by attempting to read the file, we've automatically
+        # loaded it into the database.
+        File.objects.update()
+        q = File.objects.all()
+        self.assertEqual(q.count(), 1)
+        self.assertEqual(q[0].content, "hello there")
+        
+        # Load a dynamically created file outside /media.
         test_file = files.temp.NamedTemporaryFile(
             suffix='.txt',
             dir=files.temp.gettempdir()
@@ -15,20 +54,27 @@ class DatabaseFilesTestCase(TestCase):
         t = Thing.objects.create(
             upload=files.File(test_file),
         )
-        self.assertEqual(File.objects.count(), 1)
+        self.assertEqual(File.objects.count(), 2)
         t = Thing.objects.get(pk=t.pk)
         self.assertEqual(t.upload.file.size, 10)
         self.assertEqual(t.upload.file.name[-4:], '.txt')
         self.assertEqual(t.upload.file.read(), '1234567890')
         t.upload.delete()
-        self.assertEqual(File.objects.count(), 0)
+        self.assertEqual(File.objects.count(), 1)
+        
+        # Confirm when delete a file from the database, we also delete it from
+        # the filesystem.
+        self.assertEqual(default_storage.exists('i/special/test.txt'), True)
+        default_storage.delete('i/special/test.txt')
+        self.assertEqual(default_storage.exists('i/special/test.txt'), False)
+        self.assertEqual(os.path.isfile(fqfn), False)
 
 class DatabaseFilesViewTestCase(TestCase):
     fixtures = ['test_data.json']
     
     def test_reading_file(self):
-        response = self.client.get('/1.txt')
+        self.assertEqual(File.objects.count(), 1)
+        response = self.client.get('/files/1.txt')
         self.assertEqual(response.content, '1234567890')
         self.assertEqual(response['content-type'], 'text/plain')
         self.assertEqual(unicode(response['content-length']), '10')
-

database_files/urls.py

 from django.conf.urls.defaults import *
 
 urlpatterns = patterns('',
-    url(r'^(?P<name>.+)$', 'database_files.views.serve', name='database_file'),
+    url(r'^files/(?P<name>.+)$', 'database_files.views.serve', name='database_file'),
 )

database_files/views.py

 import base64
+import os
+
 from django.http import Http404, HttpResponse
 from django.shortcuts import get_object_or_404
 from django.views.decorators.cache import cache_control
+
 import mimetypes
+
 from database_files.models import File
-import os
 
 @cache_control(max_age=86400)
 def serve(request, name):
-    pk, file_ext = os.path.splitext(name)
-    try:
-        pk = int(pk)
-    except ValueError:
-        raise Http404('Filename is not an integer')
-    f = get_object_or_404(File, pk=pk)
+    f = get_object_or_404(File, name=name)
     mimetype = mimetypes.guess_type(name)[0] or 'application/octet-stream'
-    response = HttpResponse(base64.b64decode(f.content), mimetype=mimetype)
+    response = HttpResponse(f.content, mimetype=mimetype)
     response['Content-Length'] = f.size
     return response