From c57c782eb54c783a3b1916e92a29adcd5e286d0f Mon Sep 17 00:00:00 2001 From: Tanner Collin Date: Wed, 19 Feb 2020 23:58:02 +0000 Subject: [PATCH] Change method of generating backups --- .../management/commands/generate_backups.py | 81 +++++++++++++++++++ .../management/commands/set_backup_path.py | 15 ---- apiserver/apiserver/api/models.py | 3 + apiserver/apiserver/api/views.py | 22 ++--- apiserver/apiserver/secrets.py.example | 18 +++++ apiserver/create_backup.sh | 30 ------- apiserver/delete_old_backups.sh | 9 +++ 7 files changed, 123 insertions(+), 55 deletions(-) create mode 100644 apiserver/apiserver/api/management/commands/generate_backups.py delete mode 100644 apiserver/apiserver/api/management/commands/set_backup_path.py delete mode 100755 apiserver/create_backup.sh create mode 100755 apiserver/delete_old_backups.sh diff --git a/apiserver/apiserver/api/management/commands/generate_backups.py b/apiserver/apiserver/api/management/commands/generate_backups.py new file mode 100644 index 0000000..ee8af8b --- /dev/null +++ b/apiserver/apiserver/api/management/commands/generate_backups.py @@ -0,0 +1,81 @@ +from django.core.management.base import BaseCommand, CommandError +from django.utils.timezone import now +from django.core.cache import cache + +from apiserver import secrets +from apiserver.api import models + +from uuid import uuid4 +import subprocess +import time + + +API_FOLDER = '/opt/spaceport/apiserver' +DATA_FOLDER = '/opt/spaceport/apiserver/data' +BACKUP_FOLDER = '/opt/spaceport/apiserver/backups' + +backup_id_string = lambda x: '{}\t{}\t{}'.format( + str(now()), x['name'], x['backup_id'], +) + +class Command(BaseCommand): + help = 'Generate backups.' + + def generate_backups(self): + backup_users = secrets.BACKUP_TOKENS.values() + + for user in backup_users: + models.MetaInfo.objects.update_or_create( + id=0, + defaults=dict(backup_id=backup_id_string(user)), + ) + with open(DATA_FOLDER + '/backup_user.txt', 'w') as f: + f.write(user['name'] + '\n') + with open(DATA_FOLDER + '/static/123e4567-e89b-12d3-a456-426655440000.jpg', 'w') as f: + f.write(backup_id_string(user) + '\n') + + file_name = 'spaceport-backup-{}.tar.gz'.format( + str(now().date()), + ) + + path_name = str(uuid4()) + + full_name = '{}/{}/{}'.format( + BACKUP_FOLDER, + path_name, + file_name, + ) + + mkdir_command = [ + 'mkdir', + BACKUP_FOLDER + '/' + path_name, + ] + + tar_command = [ + 'tar', + '-czf', + full_name, + '--directory', + API_FOLDER, + 'data/', + ] + + subprocess.run(mkdir_command, check=True) + subprocess.run(tar_command, check=True) + + cache.set(user['cache_key'], path_name + '/' + file_name) + + self.stdout.write('Wrote backup for: ' + user['name']) + + return len(backup_users) + + def handle(self, *args, **options): + self.stdout.write('{} - Generating backups'.format(str(now()))) + start = time.time() + + count = self.generate_backups() + self.stdout.write('Generated {} backups'.format(count)) + + self.stdout.write('Completed backups in {} s'.format( + str(time.time() - start)[:4] + )) diff --git a/apiserver/apiserver/api/management/commands/set_backup_path.py b/apiserver/apiserver/api/management/commands/set_backup_path.py deleted file mode 100644 index 8ab1c2c..0000000 --- a/apiserver/apiserver/api/management/commands/set_backup_path.py +++ /dev/null @@ -1,15 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError -from django.core.cache import cache - -class Command(BaseCommand): - help = 'Record where the last backup was saved.' - - def add_arguments(self, parser): - parser.add_argument('backup_path', type=str) - - - def handle(self, *args, **options): - backup_path = options['backup_path'] - cache.set('backup_path', backup_path) - self.stdout.write('Set backup path to: ' + backup_path) - diff --git a/apiserver/apiserver/api/models.py b/apiserver/apiserver/api/models.py index f06b476..be46722 100644 --- a/apiserver/apiserver/api/models.py +++ b/apiserver/apiserver/api/models.py @@ -122,3 +122,6 @@ class Training(models.Model): paid_date = models.DateField(blank=True, null=True) history = HistoricalRecords() + +class MetaInfo(models.Model): + backup_id = models.TextField() diff --git a/apiserver/apiserver/api/views.py b/apiserver/apiserver/api/views.py index a6a31e2..54c0f4f 100644 --- a/apiserver/apiserver/api/views.py +++ b/apiserver/apiserver/api/views.py @@ -26,7 +26,7 @@ from .permissions import ( IsAdminOrReadOnly, IsInstructorOrReadOnly ) -from .. import settings +from .. import settings, secrets # define some shortcuts Base = viewsets.GenericViewSet @@ -368,22 +368,24 @@ class StatsViewSet(viewsets.ViewSet, List): class BackupView(views.APIView): def get(self, request): - if not is_admin_director(self.request.user): + auth_token = request.META.get('HTTP_AUTHORIZATION', '') + + backup_user = secrets.BACKUP_TOKENS.get(auth_token, None) + + if not backup_user: raise exceptions.PermissionDenied() - backup_path = cache.get('backup_path') + backup_path = cache.get(backup_user['cache_key'], None) + + if not backup_path: + raise Http404 + backup_url = 'https://static.{}/backups/{}'.format( settings.PRODUCTION_HOST, backup_path, ) - if not backup_path: - raise Http404 - - if request.META['HTTP_USER_AGENT'].lower().startswith('wget'): - return redirect(backup_url) - else: - return Response(dict(url=backup_url)) + return redirect(backup_url) class PasteView(views.APIView): diff --git a/apiserver/apiserver/secrets.py.example b/apiserver/apiserver/secrets.py.example index 4501c05..836b9c6 100644 --- a/apiserver/apiserver/secrets.py.example +++ b/apiserver/apiserver/secrets.py.example @@ -26,3 +26,21 @@ LDAP_API_URL = '' # should be equal to the auth token value set in # spaceport/ldapserver/secrets.py LDAP_API_KEY = '' + +# Backup API tokens +# These tokens allow each user to download a backup of member data. +# Don't mess up the data structure! +# Tokens must be random and unique, use the output of: +# head /dev/urandom | base32 | head -c 40 +BACKUP_TOKENS = { + '': { + 'name': 'firstname_lastname', + 'backup_id': '', + 'cache_key': '', + }, + '': { + 'name': 'firstname_lastname', + 'backup_id': '', + 'cache_key': '', + }, +} diff --git a/apiserver/create_backup.sh b/apiserver/create_backup.sh deleted file mode 100755 index bca5dad..0000000 --- a/apiserver/create_backup.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# be safe -set -euf -o pipefail - - -uuid="`cat /proc/sys/kernel/random/uuid`" -date="`date -I`" - -api_folder="/opt/spaceport/apiserver" -data_folder="/opt/spaceport/apiserver/data" -backup_folder="/opt/spaceport/apiserver/backups" - -file_name="spaceport-backup-${date}.tar.gz" -path_name="${backup_folder}/${uuid}" -full_name="${path_name}/${file_name}" - -mkdir "${path_name}" -tar -czf "${full_name}" --directory "${api_folder}" data/ - -echo "Wrote backup to: ${uuid}/${file_name}" - -/opt/spaceport/apiserver/env/bin/python \ - /opt/spaceport/apiserver/manage.py \ - set_backup_path "${uuid}/${file_name}" - -# test these carefully -find "${backup_folder}" -mindepth 1 -type d -print -#find "${backup_folder}" -mindepth 1 -type d -ctime +14 -print -#find "${backup_folder}" -mindepth 1 -type d -ctime +14 -exec rm -r {} \; diff --git a/apiserver/delete_old_backups.sh b/apiserver/delete_old_backups.sh new file mode 100755 index 0000000..cd9cba3 --- /dev/null +++ b/apiserver/delete_old_backups.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# be safe +set -euf -o pipefail + +# test these carefully +#find "${backup_folder}" -mindepth 1 -type d -print +#find "${backup_folder}" -mindepth 1 -type d -ctime +14 -print +#find "${backup_folder}" -mindepth 1 -type d -ctime +14 -exec rm -r {} \;