Commit 8ee10137 authored by Arthur Shevchenko's avatar Arthur Shevchenko

Initial commit

parents
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
# C extensions
*.so
# Distribution / packaging
bin/
build/
develop-eggs/
dist/
eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
.tox/
.coverage
.cache
nosetests.xml
coverage.xml
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
.settings
# Rope
.ropeproject
# Django stuff:
*.log
*.pot
# Sphinx documentation
docs/_build/
# Local virtualenvs used for testing.
/.env*
# PIP install version files.
/=*
Copyright (c) 2009, Ben Firshman
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* The names of its contributors may not be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Django Database Files 3000
==========================
This is a storage system for Django that stores uploaded
files in the database. Files can be served from the database
(usually a bad idea), the file system, or a CDN.
Installation
------------
Simply install via pip with:
pip install django-database-files-3000
Usage
-----
In `settings.py`, add `database_files` to your `INSTALLED_APPS` and add
this line:
DEFAULT_FILE_STORAGE = 'database_files.storage.DatabaseStorage'
Note, the `upload_to` parameter is still used to synchronize the files stored
in the database with those on the file system, so new and existing fields
should still have a value that makes sense from your base media directory.
If you're using South, the initial model migrations will scan through all
existing models for `FileFields` or `ImageFields` and will automatically
load them into the database.
If for any reason you want to re-run this bulk import task, run:
python manage.py database_files_load
Additionally, if you want to export all files in the database back to the file
system, run:
python manage.py database_files_dump
Note, that when a field referencing a file is cleared, the corresponding file
in the database and on the file system will not be automatically deleted.
To delete all files in the database and file system not referenced by any model
fields, run:
python manage.py database_files_cleanup
Settings
-------
* `DB_FILES_AUTO_EXPORT_DB_TO_FS` = `True`|`False` (default `True`)
If true, when a file is uploaded or read from the database, a copy will be
exported to your media directory corresponding to the FileField's upload_to
path, just as it would with the default Django file storage.
If false, the file will only exist in the database.
* `DATABASE_FILES_URL_METHOD` = `URL_METHOD_1`|`URL_METHOD_1` (default `URL_METHOD_1`)
Defines the method to use when rendering the web-accessible URL for a file.
If `URL_METHOD_1`, assumes all files have been exported to the filesystem and
uses the path corresponding to your `settings.MEDIA_URL`.
If `URL_METHOD_2`, uses the URL bound to the `database_file` view
to dynamically lookup and serve files from the filesystem or database.
Development
-----------
You can run unittests with:
python setup.py test
You can run unittests for a specific Python version using the `pv` parameter
like:
python setup.py test --pv=3.2
\ No newline at end of file
VERSION = (0, 3, 4)
__version__ = '.'.join(map(str, VERSION))
\ No newline at end of file
from __future__ import print_function
import os
from optparse import make_option
from django.conf import settings
from django.core.files.storage import default_storage
from django.core.management.base import BaseCommand, CommandError
from django.db.models import FileField, ImageField, get_models
from database_files.models import File
class Command(BaseCommand):
args = ''
help = 'Deletes all files in the database that are not referenced by ' + \
'any model fields.'
option_list = BaseCommand.option_list + (
make_option('--dryrun',
action='store_true',
dest='dryrun',
default=False,
help='If given, only displays the names of orphaned files ' + \
'and does not delete them.'),
make_option('--filenames',
default='',
help='If given, only files with these names will be checked'),
)
def handle(self, *args, **options):
tmp_debug = settings.DEBUG
settings.DEBUG = False
names = set()
dryrun = options['dryrun']
filenames = set([
_.strip()
for _ in options['filenames'].split(',')
if _.strip()
])
try:
for model in get_models():
print('Checking model %s...' % (model,))
for field in model._meta.fields:
if not isinstance(field, (FileField, ImageField)):
continue
# Ignore records with null or empty string values.
q = {'%s__isnull'%field.name:False}
xq = {field.name:''}
subq = model.objects.filter(**q).exclude(**xq)
subq_total = subq.count()
subq_i = 0
for row in subq.iterator():
subq_i += 1
if subq_i == 1 or not subq_i % 100:
print('%i of %i' % (subq_i, subq_total))
file = getattr(row, field.name)
if file is None:
continue
if not file.name:
continue
names.add(file.name)
# Find all database files with names not in our list.
print('Finding orphaned files...')
orphan_files = File.objects.exclude(name__in=names)
if filenames:
orphan_files = orphan_files.filter(name__in=filenames)
orphan_files = orphan_files.only('name', 'size')
total_bytes = 0
orphan_total = orphan_files.count()
orphan_i = 0
print('Deleting %i orphaned files...' % (orphan_total,))
for f in orphan_files.iterator():
orphan_i += 1
if orphan_i == 1 or not orphan_i % 100:
print('%i of %i' % (orphan_i, orphan_total))
total_bytes += f.size
if dryrun:
print('File %s is orphaned.' % (f.name,))
else:
print('Deleting orphan file %s...' % (f.name,))
default_storage.delete(f.name)
print('%i total bytes in orphan files.' % total_bytes)
finally:
settings.DEBUG = tmp_debug
import os
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from database_files.models import File
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
# make_option('-w', '--overwrite', action='store_true',
# dest='overwrite', default=False,
# help='If given, overwrites any existing files.'),
)
help = 'Dumps all files in the database referenced by FileFields ' + \
'or ImageFields onto the filesystem in the directory specified by ' + \
'MEDIA_ROOT.'
def handle(self, *args, **options):
File.dump_files(verbose=True)
\ No newline at end of file
from __future__ import print_function
import os
from django.conf import settings
from django.core.files.storage import default_storage
from django.core.management.base import BaseCommand, CommandError
from django.db.models import FileField, ImageField, get_models
from optparse import make_option
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option('-m', '--models',
dest='models', default='',
help='A list of models to search for file fields. Default is all.'),
)
help = 'Loads all files on the filesystem referenced by FileFields ' + \
'or ImageFields into the database. This should only need to be ' + \
'done once, when initially migrating a legacy system.'
def handle(self, *args, **options):
show_files = int(options.get('verbosity', 1)) >= 2
all_models = [
_.lower().strip()
for _ in options.get('models', '').split()
if _.strip()
]
tmp_debug = settings.DEBUG
settings.DEBUG = False
try:
broken = 0 # Number of db records referencing missing files.
for model in get_models():
key = "%s.%s" % (model._meta.app_label, model._meta.model_name)
key = key.lower()
if all_models and key not in all_models:
continue
for field in model._meta.fields:
if not isinstance(field, (FileField, ImageField)):
continue
if show_files:
print(model.__name__, field.name)
# Ignore records with null or empty string values.
q = {'%s__isnull'%field.name:False}
xq = {field.name:''}
for row in model.objects.filter(**q).exclude(**xq):
try:
file = getattr(row, field.name)
if file is None:
continue
if not file.name:
continue
if show_files:
print("\t",file.name)
if file.path and not os.path.isfile(file.path):
if show_files:
print("Broken:",file.name)
broken += 1
continue
file.read()
row.save()
except IOError:
broken += 1
if show_files:
print('-'*80)
print('%i broken' % (broken,))
finally:
settings.DEBUG = tmp_debug
from __future__ import print_function
from optparse import make_option
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from database_files.models import File
class Command(BaseCommand):
args = '<filename 1> <filename 2> ... <filename N>'
help = 'Regenerates hashes for files. If no filenames given, ' + \
'rehashes everything.'
option_list = BaseCommand.option_list + (
# make_option('--dryrun',
# action='store_true',
# dest='dryrun',
# default=False,
# help='If given, only displays the names of orphaned files ' + \
# 'and does not delete them.'),
)
def handle(self, *args, **options):
tmp_debug = settings.DEBUG
settings.DEBUG = False
try:
q = File.objects.all()
if args:
q = q.filter(name__in=args)
total = q.count()
i = 1
for f in q.iterator():
print('%i of %i: %s' % (i, total, f.name))
f._content_hash = None
f.save()
i += 1
finally:
settings.DEBUG = tmp_debug
from django.db import models
import os
class FileManager(models.Manager):
def get_from_name(self, name):
return self.get(name=name)
\ No newline at end of file
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
]
operations = [
migrations.CreateModel(
name='File',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('name', models.CharField(unique=True, max_length=255, db_index=True)),
('size', models.PositiveIntegerField(db_index=True)),
('content', models.BinaryField()),
('created_datetime', models.DateTimeField(default=django.utils.timezone.now, verbose_name=b'Created datetime', db_index=True)),
('_content_hash', models.CharField(db_index=True, max_length=128, null=True, db_column=b'content_hash', blank=True)),
],
options={
'db_table': 'database_files_file',
},
),
]
\ No newline at end of file
from __future__ import print_function
import six
from . import settings as _settings
from django.conf import settings
from django.db import models
from django.utils import timezone
try:
from django.db.models import BinaryField
except ImportError:
from binaryfield import BinaryField
from database_files import utils
from database_files.utils import write_file, is_fresh
from database_files.manager import FileManager
class File(models.Model):
objects = FileManager()
name = models.CharField(
max_length=255,
unique=True,
blank=False,
null=False,
db_index=True)
size = models.PositiveIntegerField(
db_index=True,
blank=False,
null=False)
content = BinaryField(
blank=False,
null=False)
created_datetime = models.DateTimeField(
db_index=True,
default=timezone.now,
verbose_name="Created datetime")
_content_hash = models.CharField(
db_column='content_hash',
db_index=True,
max_length=128,
blank=True, null=True)
class Meta:
db_table = 'database_files_file'
def save(self, *args, **kwargs):
# Check for and clear old content hash.
if self.id:
old = File.objects.get(id=self.id)
if old.content != self.content:
self._content_hash = None
# Recalculate new content hash.
self.content_hash
return super(File, self).save(*args, **kwargs)
@property
def content_hash(self):
if not self._content_hash and self.content:
self._content_hash = utils.get_text_hash(self.content)
return self._content_hash
def dump(self, check_hash=False):
"""
Writes the file content to the filesystem.
If check_hash is true, clears the stored file hash and recalculates.
"""
if is_fresh(self.name, self._content_hash):
return
write_file(
self.name,
self.content,
overwrite=True)
if check_hash:
self._content_hash = None
self.save()
@classmethod
def dump_files(cls, debug=True, verbose=False):
"""
Writes all files to the filesystem.
"""
if debug:
tmp_debug = settings.DEBUG
settings.DEBUG = False
try:
q = cls.objects.only('id', 'name', '_content_hash')\
.values_list('id', 'name', '_content_hash')
total = q.count()
if verbose:
print('Checking %i total files...' % (total,))
i = 0
for (file_id, name, content_hash) in q.iterator():
i += 1
if verbose and not i % 100:
print('%i of %i' % (i, total))
if not is_fresh(name=name, content_hash=content_hash):
if verbose:
print(('File %i-%s is stale. Writing to local file '
'system...') % (file_id, name))
file = File.objects.get(id=file_id)
write_file(
file.name,
file.content,
overwrite=True)
file._content_hash = None
file.save()
finally:
if debug:
settings.DEBUG = tmp_debug
\ No newline at end of file
import os
from django.conf import settings
from django.core.urlresolvers import reverse
# If true, when file objects are created, they will be automatically copied
# to the local file system for faster serving.
settings.DB_FILES_AUTO_EXPORT_DB_TO_FS = getattr(
settings,
'DB_FILES_AUTO_EXPORT_DB_TO_FS',
True)
def URL_METHOD_1(name):
"""
Construct file URL based on media URL.
"""
return os.path.join(settings.MEDIA_URL, name)
def URL_METHOD_2(name):
"""
Construct file URL based on configured URL pattern.
"""
return reverse('database_file', kwargs={'name': name})
URL_METHODS = (
('URL_METHOD_1', URL_METHOD_1),
('URL_METHOD_2', URL_METHOD_2),
)
settings.DATABASE_FILES_URL_METHOD_NAME = getattr(
settings,
'DATABASE_FILES_URL_METHOD',
'URL_METHOD_1')
if callable(settings.DATABASE_FILES_URL_METHOD_NAME):
method = settings.DATABASE_FILES_URL_METHOD_NAME
else:
method = dict(URL_METHODS)[settings.DATABASE_FILES_URL_METHOD_NAME]
settings.DATABASE_FILES_URL_METHOD = method
settings.DB_FILES_DEFAULT_ENFORCE_ENCODING = getattr(
settings, 'DB_FILES_DEFAULT_ENFORCE_ENCODING', True)
settings.DB_FILES_DEFAULT_ENCODING = getattr(
settings,
'DB_FILES_DEFAULT_ENCODING',
'ascii')
settings.DB_FILES_DEFAULT_ERROR_METHOD = getattr(
settings, 'DB_FILES_DEFAULT_ERROR_METHOD', 'ignore')
settings.DB_FILES_DEFAULT_HASH_FN_TEMPLATE = getattr(
settings, 'DB_FILES_DEFAULT_HASH_FN_TEMPLATE', '%s.hash')
# encoding: utf-8
import datetime
from django.core.management import call_command
from django.db import models
from south.db import db
from south.v2 import SchemaMigration
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'File'
db.create_table('database_files_file', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('name', self.gf('django.db.models.fields.CharField')(unique=True, max_length=255, db_index=True)),
('size', self.gf('django.db.models.fields.PositiveIntegerField')()),
('_content', self.gf('django.db.models.fields.TextField')(db_column='content')),
))
db.send_create_signal('database_files', ['File'])
def backwards(self, orm):
# Deleting model 'File'
db.delete_table('database_files_file')
models = {
'database_files.file': {
'Meta': {'object_name': 'File'},
'_content': ('django.db.models.fields.TextField', [], {'db_column': "'content'"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}),
'size': ('django.db.models.fields.PositiveIntegerField', [], {})
}
}
complete_apps = ['database_files']
# encoding: utf-8
import datetime
from django.core.management import call_command
from django.db import models
from south.db import db
from south.v2 import DataMigration
class Migration(DataMigration):
def forwards(self, orm):
# Load any files referenced by existing models into the database.
call_command('database_files_load')
def backwards(self, orm):
import database_files
database_files.models.File.objects.all().delete()
models = {
'database_files.file': {
'Meta': {'object_name': 'File'},
'content': ('django.db.models.fields.TextField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255', 'db_index': 'True'}),
'size': ('django.db.models.fields.PositiveIntegerField', [], {})
}
}
complete_apps = ['database_files']
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
from django.utils import timezone
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'File.created_datetime'
db.add_column('database_files_file', 'created_datetime',
self.gf('django.db.models.fields.DateTimeField')(default=timezone.now, db_index=True),
keep_default=False)
# Adding field 'File._content_hash'
db.add_column('database_files_file', '_content_hash',
self.gf('django.db.models.fields.CharField')(max_length=128, null=True, db_column='content_hash', blank=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'File.created_datetime'
db.delete_column('database_files_file', 'created_datetime')
# Deleting field 'File._content_hash'