use celery to call clean_media everyday in production, update .travis.yml

.env

+# Environment variables used by `$ heroku local`
+
+DJANGO_SETTINGS_MODULE=oser_backend.settings.dev
+PORT=8000

.gitignore

@@ -7,6 +7,9 @@ virtualenv/
 *.pyc
 __pycache__/
 
+# Logs
+*.log
+
 # IDEA
 .idea
 .vscode
@@ -14,10 +17,14 @@ __pycache__/
 # databases
 *.db
 *.sqlite3
+*.rdb
 
 # private
-.env
+# .env
 
 # media and collected static files
 oser_backend/media/
 oser_backend/static/
+
+# Supervisor
+supervisord.pid

.travis.yml

@@ -5,14 +5,25 @@ python:
 
 services:
   - postgresql
+  - redis-server
+
+install:
+  - pip install -r requirements.txt
+  # Supervisor < 4 does not support python 3 but Supervisor 4 is not
+  # released to PyPI yet. => Install from Github
+  - pip install git+https://github.com/Supervisor/supervisor.git
 
 before_script:
+  # Start Celery using the supervisor config
+  - supervisord
+
   # Create local PostgreSQL database
   # NOTE: the database name (here 'oser_backend_db') must match the name
   # in one of these DATABASE_URL setting:
   # - the one set up in TravisCI environment variables
   # - the one set up in settings/default.py
   - psql -c 'create database oser_backend_db;' -U postgres
+
   # Go to the project root directory
   - cd oser_backend
   # Create and apply database migrations

Procfile

 web: sh -c 'cd oser_backend && exec gunicorn oser_backend.wsgi:application --bind 0.0.0.0:$PORT --workers 1'
+
+worker: sh -c 'cd oser_backend && exec celery -A oser_backend worker --beat -l info'

oser_backend/aws/conf.py

@@ -8,8 +8,10 @@ See:
 import os
 
 # Use S3 backends
+
 DEFAULT_FILE_STORAGE = 'aws.backends.MediaBackend'
-# Uncomment to store static files on AWS
+
+# Uncomment STATICFILES_STORAGE to store static files on AWS
 # Beware that Heroku automatically calls 'manage.py collectstatic' for
 # each deployment, and this backend does not support checking for pre-existing
 # static files on AWS : all the static files will be uploaded on each
@@ -18,6 +20,7 @@ DEFAULT_FILE_STORAGE = 'aws.backends.MediaBackend'
 # you'd have to run collectstatic manually on Heroku when necessary.
 # Since static files on the backend should not change a lot, it seems OK
 # to simply use the default file storage for static files.
+
 # STATICFILES_STORAGE = 'aws.backends.StaticBackend'
 
 # Credentials
@@ -29,13 +32,17 @@ AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME')
 # Region of the storage bucket (e.g. eu-west-1)
 AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME')
 
+
+# Do not overwrite files with the same name
+AWS_S3_FILE_OVERWRITE = False
+
+# Use the new signature version
+AWS_S3_SIGNATURE_VERSION = 's3v4'
+
 # Misc
 AWS_S3_OBJECT_PARAMETERS = {
     'CacheControl': 'max-age=86400',
 }
-# Do not overwrite files with the same name
-AWS_S3_FILE_OVERWRITE = False
-AWS_S3_SIGNATURE_VERSION = 's3v4'
 
 AWS_BASE_URL = (
     'https://{bucket}.s3.{region}.amazonaws.com/'
@@ -44,5 +51,5 @@ AWS_BASE_URL = (
 # Redefine media URL to upload/retrieve to/from S3
 MEDIA_URL = AWS_BASE_URL + 'media/'
 
-# Direct the MEDIA_ROOT to its bucket directory
+# Direct the MEDIA_ROOT to the media/ directory inside the bucket
 MEDIA_ROOT = 'media'

oser_backend/core/management/commands/clean_media.py

@@ -97,7 +97,7 @@ class Command(BaseCommand):
             return storage_files
 
         def onerror(e):
-            self.stdout.write(self.style.ERROR(e))
+            self.stdout.write(self.style.ERROR(str(e)))
 
         # Get all files from location, recursively
         for dir_root, dirs, files in os_walk(default_storage, location,

oser_backend/core/tasks.py

+"""Core Celery tasks."""
+
+from oser_backend.celery import app
+from django.core.management import call_command
+
+
+@app.task
+def clean_media():
+    """Clean unused media files."""
+    call_command('clean_media')

oser_backend/oser_backend/__init__.py

+# Celery: make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

oser_backend/oser_backend/celery.py

+"""
+Celery config for oser_backend project.
+
+It exposes the Celery application that other apps can use to
+schedule and execute tasks in the background.
+
+See:
+http://docs.celeryproject.org/en/latest/django/first-steps-with-django.html#using-celery-with-django
+"""
+import os
+from celery import Celery
+
+# set the default Django settings module for Celery
+os.environ.setdefault(
+    'DJANGO_SETTINGS_MODULE', 'oser_backend.settings.production')
+
+app = Celery('oser_backend')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load tasks from 'tasks.py' modules from all registered Django apps.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True)
+def debug_task(self):
+    """Log Celery requests information."""
+    print('Request: {0!r}'.format(self.request))

oser_backend/oser_backend/settings/common.py

+"""
+Django settings for oser_backend project.
+
+Common settings suitable for all environmebts.
+"""
+
+import os
+
+import dj_database_url
+import pymdownx.emoji
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+dn = os.path.dirname
+BASE_DIR = dn(dn(dn(os.path.abspath(__file__))))
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+# One way to do this is to store it in an environment variable on the server
+SECRET_KEY = os.environ.get('SECRET_KEY',
+                            'odfuioTvdfvkdhvjeT9659dbnkcn2332fk564jvdf034')
+
+# Admin generation settings
+ADMINS = (
+    ('Secteur Geek', 'oser.geek@gmail.com'),
+)
+ADMIN_INITIAL_PASSWORD = 'admin'  # to be changed after first login
+
+# Application definition
+
+DJANGO_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'whitenoise.runserver_nostatic',
+    'django.contrib.staticfiles',
+    'django.forms',
+]
+
+THIRD_PARTY_APPS = [
+    # Markdown integration
+    'markdownx',
+    # Django REST Framework (DRF)
+    'rest_framework',
+    'rest_framework.authtoken',
+    # DRY REST permissions (rules-based API permissions)
+    # https://github.com/dbkaplan/dry-rest-permissions
+    'dry_rest_permissions',
+    # CORS headers for Frontend integration
+    'corsheaders',
+    # Sortable models in Admin
+    'adminsortable2',
+    # Django Guardian: per object permissions
+    # https://github.com/django-guardian/django-guardian
+    'guardian',
+    # Extra Django file storage backends
+    'storages',
+]
+PROJECT_APPS = [
+    'core.apps.CoreConfig',
+    'users.apps.UsersConfig',
+    'tutoring.apps.TutoringConfig',
+    'api.apps.ApiConfig',
+    'showcase_site.apps.ShowcaseSiteConfig',
+    'visits.apps.VisitsConfig',
+]
+INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + PROJECT_APPS
+
+MIDDLEWARE = [
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.locale.LocaleMiddleware',
+    'corsheaders.middleware.CorsMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'whitenoise.middleware.WhiteNoiseMiddleware',
+    'django.middleware.security.SecurityMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'oser_backend.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [os.path.join(BASE_DIR, 'templates')],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+FORM_RENDERER = 'django.forms.renderers.TemplatesSetting'
+
+WSGI_APPLICATION = 'oser_backend.wsgi.application'
+
+# Django rest framework
+REST_FRAMEWORK = {
+    'DEFAULT_AUTHENTICATION_CLASSES': [
+        'rest_framework.authentication.TokenAuthentication',
+        # v Enable session authentication in the browsable API
+        'rest_framework.authentication.SessionAuthentication',
+    ],
+}
+
+# Security
+CORS_ORIGIN_ALLOW_ALL = True
+X_FRAME_OPTIONS = 'DENY'  # refuse to serve in an <iframe>
+
+# Pymdown-extensions Emoji configuration
+extension_configs = {
+    'emoji_index': pymdownx.emoji.twemoji,
+    'emoji_generator': pymdownx.emoji.to_png,
+    'alt': 'short',
+    'options': {
+        'attributes': {
+            'align': 'absmiddle',
+            'height': '20px',
+            'width': '20px'
+        },
+        'image_path': 'https://assets-cdn.github.com/images/icons/emoji/unicode/',
+        'non_standard_image_path': 'https://assets-cdn.github.com/images/icons/emoji/'
+    }
+}
+
+# Markdownx settings
+MARKDOWNX_MARKDOWN_EXTENSIONS = [
+    'pymdownx.emoji',
+]
+MARKDOWNX_MARKDOWN_EXTENSION_CONFIGS = {
+    'pymdownx.emoji': extension_configs,
+}
+
+# Database
+
+# Config be retrieved through the DATABASE_URL environment variable
+# DATABASE_URL format: postgres://USERNAME:PASSWORD@HOST:PORT/NAME
+DATABASES = {
+    'default': dj_database_url.config(
+        # Provide a default for dev environment
+        default='postgres://postgres:postgres@localhost:5432/oser_backend_db'),
+}
+
+
+# Authentication
+
+AUTH_USER_MODEL = 'users.User'
+AUTHENTICATION_BACKENDS = [
+    'django.contrib.auth.backends.ModelBackend',  # default
+    'guardian.backends.ObjectPermissionBackend',
+]
+
+# Password validation
+# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+# Internationalization
+# https://docs.djangoproject.com/en/2.0/topics/i18n/
+
+LANGUAGE_CODE = 'fr-fr'
+TIME_ZONE = 'Europe/Paris'
+USE_I18N = True
+USE_L10N = True
+USE_TZ = True
+
+# Static files (CSS, JavaScript, Images) and media files (user-uploaded)
+
+# Celery settings
+
+CELERY_BROKER_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379')
+CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+CELERY_ACCEPT_CONTENT = ('application/json',)
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'

oser_backend/oser_backend/settings/dev.py

-"""
-Django settings for oser_backend project.
-
-Development settings used as a base for the production settings.
-"""
+"""Development settings"""
 
 import os
+from .common import *
+from .common import BASE_DIR
 
-from django.contrib.messages import constants as messages
-
-import dj_database_url
-import pymdownx.emoji
-
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-dn = os.path.dirname
-BASE_DIR = dn(dn(dn(os.path.abspath(__file__))))
-
-# Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
-
-# SECURITY WARNING: keep the secret key used in production secret!
-# One way to do this is to store it in an environment variable on the server
-SECRET_KEY = os.environ.get('SECRET_KEY',
-                            'odfuioTvdfvkdhvjeT9659dbnkcn2332fk564jvdf034')
 DEBUG = True
 ALLOWED_HOSTS = ['localhost']
 
-ADMINS = (
-    ('admin', 'admin@oser-cs.fr'),
-)
-ADMIN_INITIAL_PASSWORD = 'admin'  # to be changed after first login
-
-# Application definition
-
-DJANGO_APPS = [
-    'django.contrib.admin',
-    'django.contrib.auth',
-    'django.contrib.contenttypes',
-    'django.contrib.sessions',
-    'django.contrib.messages',
-    'whitenoise.runserver_nostatic',
-    'django.contrib.staticfiles',
-    'django.forms',
-]
-
-THIRD_PARTY_APPS = [
-    # Markdown integration
-    'markdownx',
-    # Django REST Framework (DRF)
-    'rest_framework',
-    'rest_framework.authtoken',
-    # DRY REST permissions (rules-based API permissions)
-    # https://github.com/dbkaplan/dry-rest-permissions
-    'dry_rest_permissions',
-    # CORS headers for Frontend integration
-    'corsheaders',
-    # Sortable models in Admin
-    'adminsortable2',
-    # Django Guardian: per object permissions
-    # https://github.com/django-guardian/django-guardian
-    'guardian',
-    # Extra Django file storage backends
-    'storages',
-]
-PROJECT_APPS = [
-    'core.apps.CoreConfig',
-    'users.apps.UsersConfig',
-    'tutoring.apps.TutoringConfig',
-    'api.apps.ApiConfig',
-    'showcase_site.apps.ShowcaseSiteConfig',
-    'visits.apps.VisitsConfig',
-]
-INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + PROJECT_APPS
-
-MIDDLEWARE = [
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.middleware.locale.LocaleMiddleware',
-    'corsheaders.middleware.CorsMiddleware',
-    'django.middleware.common.CommonMiddleware',
-    'whitenoise.middleware.WhiteNoiseMiddleware',
-    'django.middleware.security.SecurityMiddleware',
-    'django.middleware.csrf.CsrfViewMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
-]
-CORS_ORIGIN_ALLOW_ALL = True
-X_FRAME_OPTIONS = 'DENY'  # refuse to serve in an <iframe>
-
-ROOT_URLCONF = 'oser_backend.urls'
-
-TEMPLATES = [
-    {
-        'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': [os.path.join(BASE_DIR, 'templates')],
-        'APP_DIRS': True,
-        'OPTIONS': {
-            'context_processors': [
-                'django.template.context_processors.debug',
-                'django.template.context_processors.request',
-                'django.contrib.auth.context_processors.auth',
-                'django.contrib.messages.context_processors.messages',
-            ],
-        },
-    },
-]
-FORM_RENDERER = 'django.forms.renderers.TemplatesSetting'
-
-WSGI_APPLICATION = 'oser_backend.wsgi.application'
-
-# Flash messages classes
-MESSAGE_TAGS = {
-    messages.INFO: 'alert alert-info alert-dismissible fade show',
-    messages.SUCCESS: 'alert alert-success alert-dismissible fade show',
-    messages.WARNING: 'alert alert-warning alert-dismissible fade show',
-    messages.ERROR: 'alert alert-danger alert-dismissible fade show',
-}
-
-# Django rest framework
-REST_FRAMEWORK = {
-    'DEFAULT_AUTHENTICATION_CLASSES': [
-        'rest_framework.authentication.TokenAuthentication',
-        # v Enable session authentication in the browsable API
-        'rest_framework.authentication.SessionAuthentication',
-    ],
-}
-
-# Pymdown-extensions Emoji configuration
-extension_configs = {
-    'emoji_index': pymdownx.emoji.twemoji,
-    'emoji_generator': pymdownx.emoji.to_png,
-    'alt': 'short',
-    'options': {
-        'attributes': {
-            'align': 'absmiddle',
-            'height': '20px',
-            'width': '20px'
-        },
-        'image_path': 'https://assets-cdn.github.com/images/icons/emoji/unicode/',
-        'non_standard_image_path': 'https://assets-cdn.github.com/images/icons/emoji/'
-    }
-}
-
-# Markdownx settings
-MARKDOWNX_MARKDOWN_EXTENSIONS = [
-    'pymdownx.emoji',
-]
-MARKDOWNX_MARKDOWN_EXTENSION_CONFIGS = {
-    'pymdownx.emoji': extension_configs,
-}
-
-# Database
-# Config be retrieved through the DATABASE_URL environment variable
-# DATABASE_URL format: postgres://USERNAME:PASSWORD@HOST:PORT/NAME
-DATABASES = {
-    'default': dj_database_url.config(
-        default='postgres://postgres:postgres@localhost:5432/oser_backend_db'),
-}
-
-# Authentication
-
-AUTH_USER_MODEL = 'users.User'
-AUTHENTICATION_BACKENDS = [
-    'django.contrib.auth.backends.ModelBackend',  # default
-    'guardian.backends.ObjectPermissionBackend',
-]
-
-# Password validation
-# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
-AUTH_PASSWORD_VALIDATORS = [
-    {
-        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
-    },
-]
-
-# Internationalization
-# https://docs.djangoproject.com/en/2.0/topics/i18n/
-
-LANGUAGE_CODE = 'fr-fr'
-TIME_ZONE = 'Europe/Paris'
-USE_I18N = True
-USE_L10N = True
-USE_TZ = True
-
 # Static files (CSS, JavaScript, Images) and media files (user-uploaded)
+
 # In development, static and media files are tied to the local filesystem.
 # In production, media files cannot be stored on Heroku and need
 # to be hosted elsewhere (e.g. AWS S3).

oser_backend/oser_backend/settings/production.py

@@ -2,12 +2,14 @@
 
 import os
 
-from .dev import *
+from celery.schedules import crontab
+
 from aws.conf import *
 
+from .common import *
+
 DEBUG = os.environ.get('DEBUG', False)
 ALLOWED_HOSTS = [
-    'florimondmanca.pythonanywhere.com',
     'localhost',
     'oser-backend.herokuapp.com',
     'oser-backend-staging.herokuapp.com',
@@ -19,3 +21,14 @@ SESSION_COOKIE_SECURE = True
 CSRF_COOKIE_SECURE = True
 SECURE_BROWSER_XSS_FILTER = True
 SECURE_SSL_REDIRECT = True
+
+
+# Celery settings
+
+CELERY_BEAT_SCHEDULE = {
+    # Clean media files every day at 22:00
+    'clean-media-every-hour': {
+        'task': 'core.tasks.clean_media',
+        'schedule': crontab(minute='0', hour='22'),
+    },
+}

oser_backend/showcase_site/migrations/0006_auto_20180407_1251.py

+# Generated by Django 2.0.3 on 2018-04-07 10:51
+
+from django.db import migrations
+import markdownx.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('showcase_site', '0005_action'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='action',
+            name='description',
+            field=markdownx.models.MarkdownxField(help_text="Un texte libre décrivant le point d'action. Markdown est supporté."),
+        ),
+    ]

oser_backend/tests/test_core/test_tasks.py

+"""Test Celery tasks."""
+
+from django.test import TestCase
+from celery.exceptions import TimeoutError
+
+from core.tasks import clean_media
+
+
+class CleanMediaTaskTest(TestCase):
+    """Test the clean_media Celery task."""
+
+    def test_run_task(self):
+        try:
+            clean_media.delay().get(timeout=2)
+        except TimeoutError as e:
+            message = str(e) + ' Is the Celery worker running?'
+            raise TimeoutError(message)

requirements.txt

@@ -16,3 +16,4 @@ gunicorn
 django-storages
 boto3
 whitenoise
+celery[redis]

supervisord.conf

+[supervisord]
+nodaemon=true
+
+[supervisorctl]
+
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:celery]
+directory = oser_backend
+command = celery -A oser_backend worker --beat -l info
+stdout_logfile=celery.log
+stderr_logfile=celery.log
+autorestart=true