...
 
Commits (52)
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
# C extensions
*.so
# Distribution / packaging
bin/
build/
develop-eggs/
dist/
eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
.tox/
.coverage
.cache
nosetests.xml
coverage.xml
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
.settings
# Rope
.ropeproject
# Django stuff:
*.log
*.pot
# Sphinx documentation
docs/_build/
# Local virtualenvs used for testing.
/.env*
# PIP install version files.
/=*
Copyright (c) 2009, Ben Firshman
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* The names of its contributors may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Django Database Files 3000
==========================
This is a storage system for Django that stores uploaded
files in the database. Files can be served from the database
(usually a bad idea), the file system, or a CDN.
Installation
------------
Simply install via pip with:
pip install django-database-files-3000
Usage
-----
In `settings.py`, add `database_files` to your `INSTALLED_APPS` and add
this line:
DEFAULT_FILE_STORAGE = 'database_files.storage.DatabaseStorage'
Note, the `upload_to` parameter is still used to synchronize the files stored
in the database with those on the file system, so new and existing fields
should still have a value that makes sense from your base media directory.
If you're using South, the initial model migrations will scan through all
existing models for `FileFields` or `ImageFields` and will automatically
load them into the database.
If for any reason you want to re-run this bulk import task, run:
python manage.py database_files_load
Additionally, if you want to export all files in the database back to the file
system, run:
python manage.py database_files_dump
Note, that when a field referencing a file is cleared, the corresponding file
in the database and on the file system will not be automatically deleted.
To delete all files in the database and file system not referenced by any model
fields, run:
python manage.py database_files_cleanup
Settings
-------
* `DB_FILES_AUTO_EXPORT_DB_TO_FS` = `True`|`False` (default `True`)
If true, when a file is uploaded or read from the database, a copy will be
exported to your media directory corresponding to the FileField's upload_to
path, just as it would with the default Django file storage.
If false, the file will only exist in the database.
* `DATABASE_FILES_URL_METHOD` = `URL_METHOD_1`|`URL_METHOD_1` (default `URL_METHOD_1`)
Defines the method to use when rendering the web-accessible URL for a file.
If `URL_METHOD_1`, assumes all files have been exported to the filesystem and
uses the path corresponding to your `settings.MEDIA_URL`.
If `URL_METHOD_2`, uses the URL bound to the `database_file` view
to dynamically lookup and serve files from the filesystem or database.
In this case, you will also need to updates your `urls.py` to include the view
that serves the files:
urlpatterns = patterns('',
# ... the rest of your URLconf goes here ...
# Serve Database Files directly
url(r'', include('database_files.urls')),
)
Development
-----------
You can run unittests with:
python setup.py test
You can run unittests for a specific Python version using the `pv` parameter
like:
python setup.py test --pv=3.2
\ No newline at end of file
VERSION = (0, 3, 4)
__version__ = '.'.join(map(str, VERSION))
\ No newline at end of file
from __future__ import print_function
import os
from optparse import make_option
from django.conf import settings
from django.core.files.storage import default_storage
from django.core.management.base import BaseCommand, CommandError
from django.db.models import FileField, ImageField, get_models
from database_files.models import File
class Command(BaseCommand):
args = ''
help = 'Deletes all files in the database that are not referenced by ' + \
'any model fields.'
option_list = BaseCommand.option_list + (
make_option('--dryrun',
action='store_true',
dest='dryrun',
default=False,
help='If given, only displays the names of orphaned files ' + \
'and does not delete them.'),
make_option('--filenames',
default='',
help='If given, only files with these names will be checked'),
)
def handle(self, *args, **options):
tmp_debug = settings.DEBUG
settings.DEBUG = False
names = set()
dryrun = options['dryrun']
filenames = set([
_.strip()
for _ in options['filenames'].split(',')
if _.strip()
])
try:
for model in get_models():
print('Checking model %s...' % (model,))
for field in model._meta.fields:
if not isinstance(field, (FileField, ImageField)):
continue
# Ignore records with null or empty string values.
q = {'%s__isnull'%field.name:False}
xq = {field.name:''}
subq = model.objects.filter(**q).exclude(**xq)
subq_total = subq.count()
subq_i = 0
for row in subq.iterator():
subq_i += 1
if subq_i == 1 or not subq_i % 100:
print('%i of %i' % (subq_i, subq_total))
file = getattr(row, field.name)
if file is None:
continue
if not file.name:
continue
names.add(file.name)
# Find all database files with names not in our list.
print('Finding orphaned files...')
orphan_files = File.objects.exclude(name__in=names)
if filenames:
orphan_files = orphan_files.filter(name__in=filenames)
orphan_files = orphan_files.only('name', 'size')
total_bytes = 0
orphan_total = orphan_files.count()
orphan_i = 0
print('Deleting %i orphaned files...' % (orphan_total,))
for f in orphan_files.iterator():
orphan_i += 1
if orphan_i == 1 or not orphan_i % 100:
print('%i of %i' % (orphan_i, orphan_total))
total_bytes += f.size
if dryrun:
print('File %s is orphaned.' % (f.name,))
else:
print('Deleting orphan file %s...' % (f.name,))
default_storage.delete(f.name)
print('%i total bytes in orphan files.' % total_bytes)
finally:
settings.DEBUG = tmp_debug
import os
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from database_files.models import File
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
# make_option('-w', '--overwrite', action='store_true',
# dest='overwrite', default=False,
# help='If given, overwrites any existing files.'),
)
help = 'Dumps all files in the database referenced by FileFields ' + \
'or ImageFields onto the filesystem in the directory specified by ' + \
'MEDIA_ROOT.'
def handle(self, *args, **options):
File.dump_files(verbose=True)
\ No newline at end of file
from __future__ import print_function
import os
from django.conf import settings
from django.core.files.storage import default_storage
from django.core.management.base import BaseCommand, CommandError
from django.db.models import FileField, ImageField, get_models
from optparse import make_option
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('-m', '--models',
dest='models', default='',
help='A list of models to search for file fields. Default is all.'),
)
help = 'Loads all files on the filesystem referenced by FileFields ' + \
'or ImageFields into the database. This should only need to be ' + \
'done once, when initially migrating a legacy system.'
def handle(self, *args, **options):
show_files = int(options.get('verbosity', 1)) >= 2
all_models = [
_.lower().strip()
for _ in options.get('models', '').split()
if _.strip()
]
tmp_debug = settings.DEBUG
settings.DEBUG = False
try:
broken = 0 # Number of db records referencing missing files.
for model in get_models():
key = "%s.%s" % (model._meta.app_label,model._meta.module_name)
key = key.lower()
if all_models and key not in all_models:
continue
for field in model._meta.fields:
if not isinstance(field, (FileField, ImageField)):
continue
if show_files:
print(model.__name__, field.name)
# Ignore records with null or empty string values.
q = {'%s__isnull'%field.name:False}
xq = {field.name:''}
for row in model.objects.filter(**q).exclude(**xq):
try:
file = getattr(row, field.name)
if file is None:
continue
if not file.name:
continue
if show_files:
print("\t",file.name)
if file.path and not os.path.isfile(file.path):
if show_files:
print("Broken:",file.name)
broken += 1
continue
file.read()
row.save()
except IOError:
broken += 1
if show_files:
print('-'*80)
print('%i broken' % (broken,))
finally:
settings.DEBUG = tmp_debug
from __future__ import print_function
from optparse import make_option
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from database_files.models import File
class Command(BaseCommand):
args = '<filename 1> <filename 2> ... <filename N>'
help = 'Regenerates hashes for files. If no filenames given, ' + \
'rehashes everything.'
option_list = BaseCommand.option_list + (
# make_option('--dryrun',
# action='store_true',
# dest='dryrun',
# default=False,
# help='If given, only displays the names of orphaned files ' + \
# 'and does not delete them.'),
)
def handle(self, *args, **options):
tmp_debug = settings.DEBUG
settings.DEBUG = False
try:
q = File.objects.all()
if args:
q = q.filter(name__in=args)
total = q.count()
i = 1
for f in q.iterator():
print('%i of %i: %s' % (i, total, f.name))
f._content_hash = None
f.save()
i += 1
finally:
settings.DEBUG = tmp_debug
from django.db import models
import os
class FileManager(models.Manager):
def get_from_name(self, name):
return self.get(name=name)
\ No newline at end of file
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
]
operations = [
migrations.CreateModel(
name='File',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('name', models.CharField(unique=True, max_length=255, db_index=True)),
('size', models.PositiveIntegerField(db_index=True)),
('_content', models.TextField(db_column='content')),
('created_datetime', models.DateTimeField(default=django.utils.timezone.now, verbose_name='Created datetime', db_index=True)),
('_content_hash', models.CharField(db_index=True, max_length=128, null=True, db_column='content_hash', blank=True)),
],
options={
'db_table': 'database_files_file',
},
),
]
from __future__ import print_function
import base64
import six
from . import settings as _settings
from django.conf import settings
from django.db import models
from django.utils import timezone
from database_files import utils
from database_files.utils import write_file, is_fresh
from database_files.manager import FileManager
class File(models.Model):
objects = FileManager()
name = models.CharField(
max_length=255,
unique=True,
blank=False,
null=False,
db_index=True)
size = models.PositiveIntegerField(
db_index=True,
blank=False,
null=False)
_content = models.TextField(db_column='content')
created_datetime = models.DateTimeField(
db_index=True,
default=timezone.now,
verbose_name="Created datetime")
_content_hash = models.CharField(
db_column='content_hash',
db_index=True,
max_length=128,
blank=True, null=True)
class Meta:
db_table = 'database_files_file'
def save(self, *args, **kwargs):
# Check for and clear old content hash.
if self.id:
old = File.objects.get(id=self.id)
if old._content != self._content:
self._content_hash = None
# Recalculate new content hash.
self.content_hash
return super(File, self).save(*args, **kwargs)
@property
def content(self):
c = self._content
if not isinstance(c, six.binary_type):
c = c.encode('utf-8')
return base64.b64decode(c)
@content.setter
def content(self, v):
self._content = base64.b64encode(v)
@property
def content_hash(self):
if not self._content_hash and self._content:
self._content_hash = utils.get_text_hash(self.content)
return self._content_hash
def dump(self, check_hash=False):
"""
Writes the file content to the filesystem.
If check_hash is true, clears the stored file hash and recalculates.
"""
if is_fresh(self.name, self._content_hash):
return
write_file(
self.name,
self.content,
overwrite=True)
if check_hash:
self._content_hash = None
self.save()
@classmethod
def dump_files(cls, debug=True, verbose=False):
"""
Writes all files to the filesystem.
"""
if debug:
tmp_debug = settings.DEBUG
settings.DEBUG = False
try:
q = cls.objects.only('id', 'name', '_content_hash')\
.values_list('id', 'name', '_content_hash')
total = q.count()
if verbose:
print('Checking %i total files...' % (total,))
i = 0
for (file_id, name, content_hash) in q.iterator():
i += 1
if verbose and not i % 100:
print('%i of %i' % (i, total))
if not is_fresh(name=name, content_hash=content_hash):
if verbose:
print(('File %i-%s is stale. Writing to local file '
'system...') % (file_id, name))
file = File.objects.get(id=file_id)
write_file(
file.name,
file.content,
overwrite=True)
file._content_hash = None
file.save()
finally:
if debug:
settings.DEBUG = tmp_debug
\ No newline at end of file
import os
from django.conf import settings
from django.urls import reverse
# If true, when file objects are created, they will be automatically copied
# to the local file system for faster serving.
settings.DB_FILES_AUTO_EXPORT_DB_TO_FS = getattr(
settings,
'DB_FILES_AUTO_EXPORT_DB_TO_FS',
True)
def URL_METHOD_1(name):
"""
Construct file URL based on media URL.
"""
return os.path.join(settings.MEDIA_URL, name)
def URL_METHOD_2(name):
"""
Construct file URL based on configured URL pattern.
"""
return reverse('database_file', kwargs={'name': name})
URL_METHODS = (
('URL_METHOD_1', URL_METHOD_1),
('URL_METHOD_2', URL_METHOD_2),
)
settings.DATABASE_FILES_URL_METHOD_NAME = getattr(
settings,
'DATABASE_FILES_URL_METHOD',
'URL_METHOD_1')
if callable(settings.DATABASE_FILES_URL_METHOD_NAME):
method = settings.DATABASE_FILES_URL_METHOD_NAME
else:
method = dict(URL_METHODS)[settings.DATABASE_FILES_URL_METHOD_NAME]
settings.DATABASE_FILES_URL_METHOD = method
settings.DB_FILES_DEFAULT_ENFORCE_ENCODING = getattr(
settings, 'DB_FILES_DEFAULT_ENFORCE_ENCODING', True)
settings.DB_FILES_DEFAULT_ENCODING = getattr(
settings,
'DB_FILES_DEFAULT_ENCODING',
'ascii')
settings.DB_FILES_DEFAULT_ERROR_METHOD = getattr(
settings, 'DB_FILES_DEFAULT_ERROR_METHOD', 'ignore')
settings.DB_FILES_DEFAULT_HASH_FN_TEMPLATE = getattr(
settings, 'DB_FILES_DEFAULT_HASH_FN_TEMPLATE', '%s.hash')
# encoding: utf-8
import datetime
from django.core.management import call_command
from django.db import models
from south.db import db
from south.v2 import SchemaMigration
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'File'
db.create_table('database_files_file', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255, db_index=True)),
('size', self.gf('django.db.models.fields.PositiveIntegerField')()),
('_content', self.gf('django.db.models.fields.TextField')(db_column='content')),
))
db.send_create_signal('database_files', ['File'])
def backwards(self, orm):
# Deleting model 'File'
db.delete_table('database_files_file')
models = {
'database_files.file': {
'Meta': {'object_name': 'File'},
'_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}),
'size': ('django.db.models.fields.PositiveIntegerField', [], {})
}
}
complete_apps = ['database_files']
# encoding: utf-8
import datetime
from django.core.management import call_command
from django.db import models
from south.db import db
from south.v2 import DataMigration
class Migration(DataMigration):
def forwards(self, orm):
# Load any files referenced by existing models into the database.
call_command('database_files_load')
def backwards(self, orm):
import database_files
database_files.models.File.objects.all().delete()
models = {
'database_files.file': {
'Meta': {'object_name': 'File'},
'_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}),
'size': ('django.db.models.fields.PositiveIntegerField', [], {})
}
}
complete_apps = ['database_files']
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
from django.utils import timezone
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'File.created_datetime'
db.add_column('database_files_file', 'created_datetime',
self.gf('django.db.models.fields.DateTimeField')(default=timezone.now, db_index=True),
keep_default=False)
# Adding field 'File._content_hash'
db.add_column('database_files_file', '_content_hash',
self.gf('django.db.models.fields.CharField')(max_length=128, null=True, db_column='content_hash', blank=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'File.created_datetime'
db.delete_column('database_files_file', 'created_datetime')
# Deleting field 'File._content_hash'
db.delete_column('database_files_file', 'content_hash')
models = {
'database_files.file': {
'Meta': {'object_name': 'File'},
'_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}),
'_content_hash': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'db_column': "'content_hash'", 'blank': 'True'}),
'created_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'timezone.now', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}),
'size': ('django.db.models.fields.PositiveIntegerField', [], {})
}
}
complete_apps = ['database_files']
\ No newline at end of file
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models
import base64
from database_files import utils
class Migration(DataMigration):
def forwards(self, orm):
"Write your forwards methods here."
File = orm['database_files.File']
q = File.objects.all()
for f in q:
f._content_hash = utils.get_text_hash_0004(base64.b64decode(f._content))
f.save()
def backwards(self, orm):
"Write your backwards methods here."
models = {
'database_files.file': {
'Meta': {'object_name': 'File'},
'_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}),
'_content_hash': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'db_column': "'content_hash'", 'blank': 'True'}),
'created_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}),
'size': ('django.db.models.fields.PositiveIntegerField', [], {})
}
}
complete_apps = ['database_files']
symmetrical = True
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding index on 'File', fields ['_content_hash']
db.create_index('database_files_file', ['content_hash'])
# Adding index on 'File', fields ['size']
db.create_index('database_files_file', ['size'])
def backwards(self, orm):
# Removing index on 'File', fields ['size']
db.delete_index('database_files_file', ['size'])
# Removing index on 'File', fields ['_content_hash']
db.delete_index('database_files_file', ['content_hash'])
models = {
'database_files.file': {
'Meta': {'object_name': 'File'},
'_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}),
'_content_hash': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '128', 'null': 'True', 'db_column': "'content_hash'", 'blank': 'True'}),
'created_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}),
'size': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'})
}
}
complete_apps = ['database_files']
\ No newline at end of file
from __future__ import print_function
import os
import six
from six import StringIO
from django.conf import settings
from django.core import files
from django.core.files.storage import FileSystemStorage
from django.urls import reverse
from database_files import models
from database_files import utils
class DatabaseStorage(FileSystemStorage):
def _generate_name(self, name, pk):
"""
Replaces the filename with the specified pk and removes any dir
"""
#dir_name, file_name = os.path.split(name)
#file_root, file_ext = os.path.splitext(file_name)
#return '%s%s' % (pk, file_name)
return name
def _open(self, name, mode='rb'):
"""
Open file with filename `name` from the database.
"""
try:
# Load file from database.
f = models.File.objects.get_from_name(name)
content = f.content
size = f.size
if settings.DB_FILES_AUTO_EXPORT_DB_TO_FS \
and not utils.is_fresh(f.name, f.content_hash):
# Automatically write the file to the filesystem
# if it's missing and exists in the database.
# This happens if we're using multiple web servers connected
# to a common databaes behind a load balancer.
# One user might upload a file from one web server, and then
# another might access if from another server.
utils.write_file(f.name, f.content)
except models.File.DoesNotExist:
# If not yet in the database, check the local file system
# and load it into the database if present.
fqfn = self.path(name)
if os.path.isfile(fqfn):
#print('Loading file into database.')
self._save(name, open(fqfn, mode))
fh = super(DatabaseStorage, self)._open(name, mode)
content = fh.read()
size = fh.size
else:
# Otherwise we don't know where the file is.
return
# Normalize the content to a new file object.
#fh = StringIO(content)
fh = six.BytesIO(content)
fh.name = name
fh.mode = mode
fh.size = size
o = files.File(fh)
return o
def _save(self, name, content):
"""
Save file with filename `name` and given content to the database.
"""
full_path = self.path(name)
try:
size = content.size
except AttributeError:
size = os.path.getsize(full_path)
content.seek(0)
content = content.read()
f = models.File.objects.create(
content=content,
size=size,
name=name,
)
# Automatically write the change to the local file system.
if settings.DB_FILES_AUTO_EXPORT_DB_TO_FS:
utils.write_file(name, content, overwrite=True)
#TODO:add callback to handle custom save behavior?
return self._generate_name(name, f.pk)
def exists(self, name):
"""
Returns true if a file with the given filename exists in the database.
Returns false otherwise.
"""
if models.File.objects.filter(name=name).exists():
return True
return super(DatabaseStorage, self).exists(name)
def delete(self, name):
"""
Deletes the file with filename `name` from the database and filesystem.
"""
try:
models.File.objects.get_from_name(name).delete()
hash_fn = utils.get_hash_fn(name)
if os.path.isfile(hash_fn):
os.remove(hash_fn)
except models.File.DoesNotExist:
pass
return super(DatabaseStorage, self).delete(name)
def url(self, name):
"""
Returns the web-accessible URL for the file with filename `name`.
"""
return settings.DATABASE_FILES_URL_METHOD(name)
def size(self, name):
"""
Returns the size of the file with filename `name` in bytes.
"""
full_path = self.path(name)
try:
return models.File.objects.get_from_name(name).size
except models.File.DoesNotExist:
return super(DatabaseStorage, self).size(name)
[
{
"pk": 1,
"model": "database_files.file",
"fields": {
"name": "1.txt",
"_content": "MTIzNDU2Nzg5MA==",
"size": 10
}
}
]
\ No newline at end of file
This diff is collapsed.
from django.db import models
class Thing(models.Model):
upload = models.FileField(upload_to='i/special')
import os, sys
PROJECT_DIR = os.path.dirname(__file__)
DATABASES = {
'default':{
'ENGINE': 'django.db.backends.sqlite3',
# Don't do this. It dramatically slows down the test.
# 'NAME': '/tmp/database_files.db',
# 'TEST_NAME': '/tmp/database_files.db',
}
}
ROOT_URLCONF = 'database_files.urls'
INSTALLED_APPS = [
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.sites',
'database_files',
'database_files.tests',
'south',
]
DEFAULT_FILE_STORAGE = 'database_files.storage.DatabaseStorage'
MEDIA_ROOT = os.path.join(PROJECT_DIR, 'media')
# Run our South migrations during unittesting.
SOUTH_TESTS_MIGRATE = True
USE_TZ = True
SECRET_KEY = 'secret'
# -*- coding: utf-8 -*-
import os
import shutil
import six
from six import StringIO
from django.core import files
from django.test import TestCase
from django.core.files.storage import default_storage
from database_files.models import File
from database_files.tests.models import Thing
from database_files import utils
DIR = os.path.abspath(os.path.split(__file__)[0])
class DatabaseFilesTestCase(TestCase):
def setUp(self):
self.media_dir = os.path.join(DIR, 'media/i/special')
if os.path.isdir(self.media_dir):
shutil.rmtree(self.media_dir)
os.makedirs(self.media_dir)
def test_adding_file(self):
# Create default thing storing reference to file
# in the local media directory.
test_fqfn = os.path.join(self.media_dir, 'test.txt')
open(test_fqfn,'w').write('hello there')
o1 = o = Thing()
test_fn = 'i/special/test.txt'
o.upload = test_fn
o.save()
id = o.id
# Confirm thing was saved.
Thing.objects.update()
q = Thing.objects.all()
self.assertEqual(q.count(), 1)
self.assertEqual(q[0].upload.name, test_fn)
# Confirm the file only exists on the file system
# and hasn't been loaded into the database.
q = File.objects.all()
self.assertEqual(q.count(), 0)
# Verify we can read the contents of thing.
o = Thing.objects.get(id=id)
self.assertEqual(o.upload.read(), b"hello there")
# Verify that by attempting to read the file, we've automatically
# loaded it into the database.
File.objects.update()
q = File.objects.all()
self.assertEqual(q.count(), 1)
self.assertEqual(q[0].content, b"hello there")
# Load a dynamically created file outside /media.
test_file = files.temp.NamedTemporaryFile(
suffix='.txt',
dir=files.temp.gettempdir()
)
test_file.write(b'1234567890')
test_file.seek(0)
t = Thing.objects.create(
upload=files.File(test_file),
)
self.assertEqual(File.objects.count(), 2)
t = Thing.objects.get(pk=t.pk)
self.assertEqual(t.upload.file.size, 10)
self.assertEqual(t.upload.file.name[-4:], '.txt')
self.assertEqual(t.upload.file.read(), b'1234567890')
t.upload.delete()
self.assertEqual(File.objects.count(), 1)
# Delete file from local filesystem and re-export it from the database.
self.assertEqual(os.path.isfile(test_fqfn), True)
os.remove(test_fqfn)
self.assertEqual(os.path.isfile(test_fqfn), False)
o1.upload.read() # This forces the re-export to the filesystem.
self.assertEqual(os.path.isfile(test_fqfn), True)
# This dumps all files to the filesystem.
File.dump_files()
# Confirm when delete a file from the database, we also delete it from
# the filesystem.
self.assertEqual(default_storage.exists('i/special/test.txt'), True)
default_storage.delete('i/special/test.txt')
self.assertEqual(default_storage.exists('i/special/test.txt'), False)
self.assertEqual(os.path.isfile(test_fqfn), False)
def test_hash(self):
verbose = 1
# Create test file.
image_content = open(os.path.join(DIR, 'fixtures/test_image.png'), 'rb').read()
fqfn = os.path.join(self.media_dir, 'image.png')
open(fqfn, 'wb').write(image_content)
# Calculate hash from various sources and confirm they all match.
expected_hash = '35830221efe45ab0dc3d91ca23c29d2d3c20d00c9afeaa096ab256ec322a7a0b3293f07a01377e31060e65b4e5f6f8fdb4c0e56bc586bba5a7ab3e6d6d97a192'
h = utils.get_text_hash(image_content)
self.assertEqual(h, expected_hash)
h = utils.get_file_hash(fqfn)
self.assertEqual(h, expected_hash)
h = utils.get_text_hash(open(fqfn, 'rb').read())
self.assertEqual(h, expected_hash)
# h = utils.get_text_hash(open(fqfn, 'r').read())#not supported in py3
# self.assertEqual(h, expected_hash)
# Create test file.
if six.PY3:
image_content = six.text_type('aあä')#, encoding='utf-8')
else:
image_content = six.text_type('aあä', encoding='utf-8')
fqfn = os.path.join(self.media_dir, 'test.txt')
open(fqfn, 'wb').write(image_content.encode('utf-8'))
expected_hash = '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75'
h = utils.get_text_hash(image_content)
self.assertEqual(h, expected_hash)
h = utils.get_file_hash(fqfn)
self.assertEqual(h, expected_hash)
h = utils.get_text_hash(open(fqfn, 'rb').read())
self.assertEqual(h, expected_hash)
# h = utils.get_text_hash(open(fqfn, 'r').read())
# self.assertEqual(h, expected_hash)
class DatabaseFilesViewTestCase(TestCase):
fixtures = ['test_data.json']
def test_reading_file(self):
self.assertEqual(File.objects.count(), 1)
response = self.client.get('/files/1.txt')
if hasattr(response, 'streaming_content'):
content = list(response.streaming_content)[0]
else:
content = response.content
self.assertEqual(content, b'1234567890')
self.assertEqual(response['content-type'], 'text/plain')
self.assertEqual(response['content-length'], '10')
from django.urls import re_path
from database_files import views
urlpatterns = [
# url(r'^files/(?P<name>.+)$',
# 'database_files.views.serve',
# name='database_file'),
re_path(r'^files/(?P<name>.+)$',
views.serve_mixed,
name='database_file'),
]
#from grp import getgrnam
#from pwd import getpwnam
import os
import hashlib
import six
from django.conf import settings
def is_fresh(name, content_hash):
"""
Returns true if the file exists on the local filesystem and matches the
content in the database. Returns false otherwise.
"""
if not content_hash:
return False
# Check that the actual file exists.
fqfn = os.path.join(settings.MEDIA_ROOT, name)
fqfn = os.path.normpath(fqfn)
if not os.path.isfile(fqfn):
return False
# Check for cached hash file.
hash_fn = get_hash_fn(name)
if os.path.isfile(hash_fn):
return open(hash_fn).read().strip() == content_hash
# Otherwise, calculate the hash of the local file.
fqfn = os.path.join(settings.MEDIA_ROOT, name)
fqfn = os.path.normpath(fqfn)
if not os.path.isfile(fqfn):
return False
local_content_hash = get_file_hash(fqfn)
return local_content_hash == content_hash
def get_hash_fn(name):
"""
Returns the filename for the hash file.
"""
fqfn = os.path.join(settings.MEDIA_ROOT, name)
fqfn = os.path.normpath(fqfn)
dirs,fn = os.path.split(fqfn)
if not os.path.isdir(dirs):
os.makedirs(dirs)
fqfn_parts = os.path.split(fqfn)
hash_fn = os.path.join(fqfn_parts[0],
settings.DB_FILES_DEFAULT_HASH_FN_TEMPLATE % fqfn_parts[1])
return hash_fn
def write_file(name, content, overwrite=False):
"""
Writes the given content to the relative filename under the MEDIA_ROOT.
"""
fqfn = os.path.join(settings.MEDIA_ROOT, name)
fqfn = os.path.normpath(fqfn)
if os.path.isfile(fqfn) and not overwrite:
return
dirs,fn = os.path.split(fqfn)
if not os.path.isdir(dirs):
os.makedirs(dirs)
open(fqfn, 'wb').write(content)
# Cache hash.
hash = get_file_hash(fqfn)
hash_fn = get_hash_fn(name)
try:
# Write out bytes in Python3.
value = six.binary_type(hash, 'utf-8')
except TypeError:
value = hash
open(hash_fn, 'wb').write(value)
# Set ownership and permissions.
uname = getattr(settings, 'DATABASE_FILES_USER', None)
gname = getattr(settings, 'DATABASE_FILES_GROUP', None)
if gname:
gname = ':'+gname
if uname:
os.system('chown -RL %s%s "%s"' % (uname, gname, dirs))
# Set permissions.
perms = getattr(settings, 'DATABASE_FILES_PERMS', None)
if perms:
os.system('chmod -R %s "%s"' % (perms, dirs))
def get_file_hash(fin,
force_encoding=settings.DB_FILES_DEFAULT_ENFORCE_ENCODING,
encoding=settings.DB_FILES_DEFAULT_ENCODING,
errors=settings.DB_FILES_DEFAULT_ERROR_METHOD,
chunk_size=128):
"""
Iteratively builds a file hash without loading the entire file into memory.
"""
if isinstance(fin, six.string_types):
fin = open(fin, 'rb')
h = hashlib.sha512()
while 1:
text = fin.read(chunk_size)
if not text:
break
if force_encoding:
if not isinstance(text, six.text_type):
text = six.text_type(text, encoding=encoding, errors=errors)
h.update(text.encode(encoding, errors))
else:
h.update(text)
return h.hexdigest()
def get_text_hash_0004(text):
"""
Returns the hash of the given text.
"""
h = hashlib.sha512()
if not isinstance(text, six.text_type):
text = six.text_type(text, encoding='utf-8', errors='replace')
h.update(text.encode('utf-8', 'replace'))
return h.hexdigest()
def get_text_hash(text,
force_encoding=settings.DB_FILES_DEFAULT_ENFORCE_ENCODING,
encoding=settings.DB_FILES_DEFAULT_ENCODING,
errors=settings.DB_FILES_DEFAULT_ERROR_METHOD):
"""
Returns the hash of the given text.
"""
h = hashlib.sha512()
if force_encoding:
if not isinstance(text, six.text_type):
text = six.text_type(text, encoding=encoding, errors=errors)
h.update(text.encode(encoding, errors))
else:
h.update(text)
return h.hexdigest()
import base64
import os
from django.conf import settings
from django.http import Http404, HttpResponse
from django.shortcuts import get_object_or_404
from django.views.decorators.cache import cache_control
from django.views.static import serve as django_serve
import mimetypes
from database_files.models import File
@cache_control(max_age=86400)
def serve(request, name):
"""
Retrieves the file from the database.
"""
f = get_object_or_404(File, name=name)
f.dump()
mimetype = mimetypes.guess_type(name)[0] or 'application/octet-stream'
response = HttpResponse(f.content, content_type=mimetype)
response['Content-Length'] = f.size
return response
def serve_mixed(request, *args, **kwargs):
"""
First attempts to serve the file from the filesystem,
then tries the database.
"""
name = kwargs.get('name') or kwargs.get('path')
document_root = kwargs.get('document_root')
document_root = document_root or settings.MEDIA_ROOT
try:
# First attempt to serve from filesystem.
return django_serve(request, name, document_root)
except Http404:
# Then try serving from database.
return serve(request, name)
\ No newline at end of file
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
from setuptools import setup, find_packages, Command
import database_files
try:
from pypandoc import convert
read_md = lambda f: convert(f, 'rst')
except:
print("Warning: pypandoc module not found, could not convert "
"Markdown to RST")
read_md = lambda f: open(f, 'r').read()
def get_reqs(test=False, pv=None):
reqs = [
'Django>=1.4',
'six>=1.7.2',
]
if test:
#TODO:remove once Django 1.7 south integration becomes main-stream?
if not pv:
reqs.append('South>=1.0')
elif pv <= 3.2:
# Note, South dropped Python3 support after 0.8.4...
reqs.append('South==0.8.4')
return reqs
class TestCommand(Command):
description = "Runs unittests."
user_options = [
('name=', None,
'Name of the specific test to run.'),
('virtual-env-dir=', None,
'The location of the virtual environment to use.'),
('pv=', None,
'The version of Python to use. e.g. 2.7 or 3'),
]
def initialize_options(self):
self.name = None
self.virtual_env_dir = '.env%s'
self.pv = 0