Commit 84cdd2fc authored by chrisspen's avatar chrisspen
Browse files

Modified database storage to fallback to file storage. Added management

commands to handle bulk load and dump of files to/from the filesystem.
parent 3c3ebeed
*.pyc
/build
.project
.pydevproject
.settings
\ No newline at end of file
import os
from django.conf import settings
from django.core.files.storage import default_storage
from django.core.management.base import BaseCommand, CommandError
from django.db.models import FileField, ImageField
from database_files.models import File
from optparse import make_option
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('-w', '--overwrite', action='store_true',
dest='overwrite', default=False,
help='If given, overwrites any existing files.'),
)
help = 'Dumps all files in the database referenced by FileFields ' + \
'or ImageFields onto the filesystem in the directory specified by ' + \
'MEDIA_ROOT.'
def handle(self, *args, **options):
tmp_debug = settings.DEBUG
settings.DEBUG = False
try:
q = File.objects.all()
total = q.count()
i = 0
for file in q:
i += 1
print '%i of %i' % (i, total)
fqfn = os.path.join(settings.MEDIA_ROOT, file.name)
fqfn = os.path.normpath(fqfn)
if os.path.isfile(fqfn) and not options['overwrite']:
continue
dirs,fn = os.path.split(fqfn)
if not os.path.isdir(dirs):
os.makedirs(dirs)
open(fqfn, 'wb').write(file.content)
finally:
settings.DEBUG = tmp_debug
\ No newline at end of file
import os
from django.conf import settings
from django.core.files.storage import default_storage
from django.core.management.base import BaseCommand, CommandError
from django.db.models import FileField, ImageField
from optparse import make_option
class Command(BaseCommand):
args = ''
help = 'Loads all files on the filesystem referenced by FileFields ' + \
'or ImageFields into the database. This should only need to be ' + \
'done once, when initially migrating a legacy system.'
def handle(self, *args, **options):
tmp_debug = settings.DEBUG
settings.DEBUG = False
try:
broken = 0 # Number of db records referencing missing files.
from django.db.models import get_models
for model in get_models():
for field in model._meta.fields:
if not isinstance(field, (FileField, ImageField)):
continue
print model.__name__, field.name
# Ignore records with null or empty string values.
q = {'%s__isnull'%field.name:False}
xq = {field.name:''}
for row in model.objects.filter(**q).exclude(**xq):
try:
file = getattr(row, field.name)
if file is None:
continue
if not file.name:
continue
if file.path and not os.path.isfile(file.path):
broken += 1
continue
file.read()
row.save()
except IOError:
broken += 1
print '-'*80
print '%i broken' % (broken,)
finally:
settings.DEBUG = tmp_debug
......@@ -3,4 +3,6 @@ import os
class FileManager(models.Manager):
def get_from_name(self, name):
return self.get(pk=os.path.splitext(os.path.split(name)[1])[0])
# print 'name:',name
# return self.get(pk=os.path.splitext(os.path.split(name)[1])[0])
return self.get(name=name)
\ No newline at end of file
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
from database_files.models import *
class Migration:
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'File'
db.create_table('database_files_file', (
('id', orm['database_files.File:id']),
('content', orm['database_files.File:content']),
('size', orm['database_files.File:size']),
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255, db_index=True)),
('size', self.gf('django.db.models.fields.PositiveIntegerField')()),
('_content', self.gf('django.db.models.fields.TextField')(db_column='content')),
))
db.send_create_signal('database_files', ['File'])
def backwards(self, orm):
# Deleting model 'File'
db.delete_table('database_files_file')
models = {
'database_files.file': {
'content': ('django.db.models.fields.TextField', [], {}),
'Meta': {'object_name': 'File'},
'_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'size': ('django.db.models.fields.IntegerField', [], {})
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}),
'size': ('django.db.models.fields.PositiveIntegerField', [], {})
}
}
complete_apps = ['database_files']
import base64
from django.db import models
from database_files.manager import FileManager
class File(models.Model):
content = models.TextField()
size = models.IntegerField()
objects = FileManager()
name = models.CharField(
max_length=255,
unique=True,
blank=False,
null=False,
db_index=True)
size = models.PositiveIntegerField(
blank=False,
null=False)
_content = models.TextField(db_column='content')
@property
def content(self):
return base64.b64decode(self._content)
@content.setter
def content(self, v):
self._content = base64.b64encode(v)
\ No newline at end of file
import base64
from database_files import models
from django.core import files
from django.core.files.storage import Storage
from django.core.urlresolvers import reverse
import os
import StringIO
class DatabaseStorage(Storage):
from django.conf import settings
from django.core import files
from django.core.files.storage import FileSystemStorage
from django.core.urlresolvers import reverse
from database_files import models
class DatabaseStorage(FileSystemStorage):
def _generate_name(self, name, pk):
"""
Replaces the filename with the specified pk and removes any dir
"""
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
return '%s%s' % (pk, file_ext)
#dir_name, file_name = os.path.split(name)
#file_root, file_ext = os.path.splitext(file_name)
#return '%s%s' % (pk, file_name)
return name
def _open(self, name, mode='rb'):
"""
Open file with filename `name` from the database.
"""
try:
# Load file from database.
f = models.File.objects.get_from_name(name)
content = f.content
size = f.size
except models.File.DoesNotExist:
return None
fh = StringIO.StringIO(base64.b64decode(f.content))
# If not yet in the database, check the local file system
# and load it into the database if present.
fqfn = self.path(name)
if os.path.isfile(fqfn):
self._save(name, open(fqfn, mode))
fh = super(DatabaseStorage, self)._open(name, mode)
content = fh.read()
size = fh.size
else:
# Otherwise we don't know where the file is.
return
# Normalize the content to a new file object.
fh = StringIO.StringIO(content)
fh.name = name
fh.mode = mode
fh.size = f.size
return files.File(fh)
fh.size = size
o = files.File(fh)
return o
def _save(self, name, content):
"""
Save file with filename `name` and given content to the database.
"""
full_path = self.path(name)
try:
size = content.size
except AttributeError:
size = os.path.getsize(full_path)
f = models.File.objects.create(
content=base64.b64encode(content.read()),
size=content.size,
content=content.read(),
size=size,
name=name,
)
return self._generate_name(name, f.pk)
def exists(self, name):
"""
We generate a new filename for each file, so it will never already
exist.
Returns true if a file with the given filename exists in the database.
Returns false otherwise.
"""
return False
if models.File.objects.filter(name=name).count() > 0:
return True
return super(DatabaseStorage, self).exists(name)
def delete(self, name):
"""
Deletes the file with filename `name` from the database and filesystem.
"""
full_path = self.path(name)
try:
models.File.objects.get_from_name(name).delete()
except models.File.DoesNotExist:
pass
return super(DatabaseStorage, self).delete(name)
def url(self, name):
return reverse('database_file', kwargs={'name': name})
"""
Returns the web-accessible URL for the file with filename `name`.
"""
return os.path.join(settings.MEDIA_URL, name)
#return reverse('database_file', kwargs={'name': name})
def size(self, name):
"""
Returns the size of the file with filename `name` in bytes.
"""
full_path = self.path(name)
try:
return models.File.objects.get_from_name(name).size
except models.File.DoesNotExist:
return 0
return super(DatabaseStorage, self).size(name)
[
{
"pk": 1,
"model": "database_files.file",
"pk": 1,
"model": "database_files.file",
"fields": {
"content": "MTIzNDU2Nzg5MA==",
"name": "1.txt",
"_content": "MTIzNDU2Nzg5MA==",
"size": 10
}
}
]
]
\ No newline at end of file
from django.db import models
class Thing(models.Model):
upload = models.FileField(upload_to='not required')
upload = models.FileField(upload_to='i/special')
DATABASE_ENGINE = 'sqlite3'
import os, sys
PROJECT_DIR = os.path.dirname(__file__)
DATABASES = {
'default':{
'ENGINE': 'django.db.backends.sqlite3',
# Don't do this. It dramatically slows down the test.
# 'NAME': '/tmp/database_files.db',
# 'TEST_NAME': '/tmp/database_files.db',
}
}
ROOT_URLCONF = 'database_files.urls'
INSTALLED_APPS = [
'django.contrib.auth',
......@@ -9,3 +18,4 @@ INSTALLED_APPS = [
'database_files.tests',
]
DEFAULT_FILE_STORAGE = 'database_files.storage.DatabaseStorage'
MEDIA_ROOT = os.path.join(PROJECT_DIR, 'media')
\ No newline at end of file
import os
import StringIO
from django.core import files
from django.test import TestCase
from django.core.files.storage import default_storage
from database_files.models import File
from database_files.tests.models import Thing
import StringIO
DIR = os.path.abspath(os.path.split(__file__)[0])
class DatabaseFilesTestCase(TestCase):
def test_adding_file(self):
# Create default thing storing reference to file
# in the local media directory.
fqfn = os.path.join(DIR,'media/i/special/test.txt')
open(fqfn,'w').write('hello there')
o = Thing()
o.upload = 'i/special/test.txt'
o.save()
id = o.id
# Confirm thing was saved.
Thing.objects.update()
q = Thing.objects.all()
self.assertEqual(q.count(), 1)
self.assertEqual(q[0].upload.name, 'i/special/test.txt')
# Confirm the file only exists on the file system
# and hasn't been loaded into the database.
q = File.objects.all()
self.assertEqual(q.count(), 0)
# Verify we can read the contents of thing.
o = Thing.objects.get(id=id)
self.assertEqual(o.upload.read(), "hello there")
# Verify that by attempting to read the file, we've automatically
# loaded it into the database.
File.objects.update()
q = File.objects.all()
self.assertEqual(q.count(), 1)
self.assertEqual(q[0].content, "hello there")
# Load a dynamically created file outside /media.
test_file = files.temp.NamedTemporaryFile(
suffix='.txt',
dir=files.temp.gettempdir()
......@@ -15,20 +54,27 @@ class DatabaseFilesTestCase(TestCase):
t = Thing.objects.create(
upload=files.File(test_file),
)
self.assertEqual(File.objects.count(), 1)
self.assertEqual(File.objects.count(), 2)
t = Thing.objects.get(pk=t.pk)
self.assertEqual(t.upload.file.size, 10)
self.assertEqual(t.upload.file.name[-4:], '.txt')
self.assertEqual(t.upload.file.read(), '1234567890')
t.upload.delete()
self.assertEqual(File.objects.count(), 0)
self.assertEqual(File.objects.count(), 1)
# Confirm when delete a file from the database, we also delete it from
# the filesystem.
self.assertEqual(default_storage.exists('i/special/test.txt'), True)
default_storage.delete('i/special/test.txt')
self.assertEqual(default_storage.exists('i/special/test.txt'), False)
self.assertEqual(os.path.isfile(fqfn), False)
class DatabaseFilesViewTestCase(TestCase):
fixtures = ['test_data.json']
def test_reading_file(self):
response = self.client.get('/1.txt')
self.assertEqual(File.objects.count(), 1)
response = self.client.get('/files/1.txt')
self.assertEqual(response.content, '1234567890')
self.assertEqual(response['content-type'], 'text/plain')
self.assertEqual(unicode(response['content-length']), '10')
from django.conf.urls.defaults import *
urlpatterns = patterns('',
url(r'^(?P<name>.+)$', 'database_files.views.serve', name='database_file'),
url(r'^files/(?P<name>.+)$', 'database_files.views.serve', name='database_file'),
)
import base64
import os
from django.http import Http404, HttpResponse
from django.shortcuts import get_object_or_404
from django.views.decorators.cache import cache_control
import mimetypes
from database_files.models import File
import os
@cache_control(max_age=86400)
def serve(request, name):
pk, file_ext = os.path.splitext(name)
try:
pk = int(pk)
except ValueError:
raise Http404('Filename is not an integer')
f = get_object_or_404(File, pk=pk)
f = get_object_or_404(File, name=name)
mimetype = mimetypes.guess_type(name)[0] or 'application/octet-stream'
response = HttpResponse(base64.b64decode(f.content), mimetype=mimetype)
response = HttpResponse(f.content, mimetype=mimetype)
response['Content-Length'] = f.size
return response
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment