Change method of generating backups

This commit is contained in:
Tanner Collin 2020-02-19 23:58:02 +00:00
parent fe758271f2
commit c57c782eb5
7 changed files with 123 additions and 55 deletions

View File

@ -0,0 +1,81 @@
from django.core.management.base import BaseCommand, CommandError
from django.utils.timezone import now
from django.core.cache import cache
from apiserver import secrets
from apiserver.api import models
from uuid import uuid4
import subprocess
import time
API_FOLDER = '/opt/spaceport/apiserver'
DATA_FOLDER = '/opt/spaceport/apiserver/data'
BACKUP_FOLDER = '/opt/spaceport/apiserver/backups'
backup_id_string = lambda x: '{}\t{}\t{}'.format(
str(now()), x['name'], x['backup_id'],
)
class Command(BaseCommand):
help = 'Generate backups.'
def generate_backups(self):
backup_users = secrets.BACKUP_TOKENS.values()
for user in backup_users:
models.MetaInfo.objects.update_or_create(
id=0,
defaults=dict(backup_id=backup_id_string(user)),
)
with open(DATA_FOLDER + '/backup_user.txt', 'w') as f:
f.write(user['name'] + '\n')
with open(DATA_FOLDER + '/static/123e4567-e89b-12d3-a456-426655440000.jpg', 'w') as f:
f.write(backup_id_string(user) + '\n')
file_name = 'spaceport-backup-{}.tar.gz'.format(
str(now().date()),
)
path_name = str(uuid4())
full_name = '{}/{}/{}'.format(
BACKUP_FOLDER,
path_name,
file_name,
)
mkdir_command = [
'mkdir',
BACKUP_FOLDER + '/' + path_name,
]
tar_command = [
'tar',
'-czf',
full_name,
'--directory',
API_FOLDER,
'data/',
]
subprocess.run(mkdir_command, check=True)
subprocess.run(tar_command, check=True)
cache.set(user['cache_key'], path_name + '/' + file_name)
self.stdout.write('Wrote backup for: ' + user['name'])
return len(backup_users)
def handle(self, *args, **options):
self.stdout.write('{} - Generating backups'.format(str(now())))
start = time.time()
count = self.generate_backups()
self.stdout.write('Generated {} backups'.format(count))
self.stdout.write('Completed backups in {} s'.format(
str(time.time() - start)[:4]
))

View File

@ -1,15 +0,0 @@
from django.core.management.base import BaseCommand, CommandError
from django.core.cache import cache
class Command(BaseCommand):
help = 'Record where the last backup was saved.'
def add_arguments(self, parser):
parser.add_argument('backup_path', type=str)
def handle(self, *args, **options):
backup_path = options['backup_path']
cache.set('backup_path', backup_path)
self.stdout.write('Set backup path to: ' + backup_path)

View File

@ -122,3 +122,6 @@ class Training(models.Model):
paid_date = models.DateField(blank=True, null=True) paid_date = models.DateField(blank=True, null=True)
history = HistoricalRecords() history = HistoricalRecords()
class MetaInfo(models.Model):
backup_id = models.TextField()

View File

@ -26,7 +26,7 @@ from .permissions import (
IsAdminOrReadOnly, IsAdminOrReadOnly,
IsInstructorOrReadOnly IsInstructorOrReadOnly
) )
from .. import settings from .. import settings, secrets
# define some shortcuts # define some shortcuts
Base = viewsets.GenericViewSet Base = viewsets.GenericViewSet
@ -368,22 +368,24 @@ class StatsViewSet(viewsets.ViewSet, List):
class BackupView(views.APIView): class BackupView(views.APIView):
def get(self, request): def get(self, request):
if not is_admin_director(self.request.user): auth_token = request.META.get('HTTP_AUTHORIZATION', '')
backup_user = secrets.BACKUP_TOKENS.get(auth_token, None)
if not backup_user:
raise exceptions.PermissionDenied() raise exceptions.PermissionDenied()
backup_path = cache.get('backup_path') backup_path = cache.get(backup_user['cache_key'], None)
if not backup_path:
raise Http404
backup_url = 'https://static.{}/backups/{}'.format( backup_url = 'https://static.{}/backups/{}'.format(
settings.PRODUCTION_HOST, settings.PRODUCTION_HOST,
backup_path, backup_path,
) )
if not backup_path:
raise Http404
if request.META['HTTP_USER_AGENT'].lower().startswith('wget'):
return redirect(backup_url) return redirect(backup_url)
else:
return Response(dict(url=backup_url))
class PasteView(views.APIView): class PasteView(views.APIView):

View File

@ -26,3 +26,21 @@ LDAP_API_URL = ''
# should be equal to the auth token value set in # should be equal to the auth token value set in
# spaceport/ldapserver/secrets.py # spaceport/ldapserver/secrets.py
LDAP_API_KEY = '' LDAP_API_KEY = ''
# Backup API tokens
# These tokens allow each user to download a backup of member data.
# Don't mess up the data structure!
# Tokens must be random and unique, use the output of:
# head /dev/urandom | base32 | head -c 40
BACKUP_TOKENS = {
'<token>': {
'name': 'firstname_lastname',
'backup_id': '<token>',
'cache_key': '<token>',
},
'<token>': {
'name': 'firstname_lastname',
'backup_id': '<token>',
'cache_key': '<token>',
},
}

View File

@ -1,30 +0,0 @@
#!/bin/bash
# be safe
set -euf -o pipefail
uuid="`cat /proc/sys/kernel/random/uuid`"
date="`date -I`"
api_folder="/opt/spaceport/apiserver"
data_folder="/opt/spaceport/apiserver/data"
backup_folder="/opt/spaceport/apiserver/backups"
file_name="spaceport-backup-${date}.tar.gz"
path_name="${backup_folder}/${uuid}"
full_name="${path_name}/${file_name}"
mkdir "${path_name}"
tar -czf "${full_name}" --directory "${api_folder}" data/
echo "Wrote backup to: ${uuid}/${file_name}"
/opt/spaceport/apiserver/env/bin/python \
/opt/spaceport/apiserver/manage.py \
set_backup_path "${uuid}/${file_name}"
# test these carefully
find "${backup_folder}" -mindepth 1 -type d -print
#find "${backup_folder}" -mindepth 1 -type d -ctime +14 -print
#find "${backup_folder}" -mindepth 1 -type d -ctime +14 -exec rm -r {} \;

View File

@ -0,0 +1,9 @@
#!/bin/bash
# be safe
set -euf -o pipefail
# test these carefully
#find "${backup_folder}" -mindepth 1 -type d -print
#find "${backup_folder}" -mindepth 1 -type d -ctime +14 -print
#find "${backup_folder}" -mindepth 1 -type d -ctime +14 -exec rm -r {} \;