tanner/spaceport
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'load_more'

Browse Source
This commit is contained in:
Tanner Collin
2021-12-08 22:09:06 +00:00
parent cac6a7223f fd83d171e9
commit bf5c300c6c
17 changed files with 461 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
apiserver/apiserver/api/serializers.py
View File

@@ -3,6 +3,7 @@ logger = logging.getLogger(__name__)
from django.contrib.auth.models import User, Group
from django.shortcuts import get_object_or_404
from django.utils.timezone import now
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
from rest_framework.validators import UniqueValidator
@@ -10,7 +11,7 @@ from rest_auth.registration.serializers import RegisterSerializer
from rest_auth.serializers import PasswordChangeSerializer, PasswordResetSerializer, PasswordResetConfirmSerializer, LoginSerializer
from rest_auth.serializers import UserDetailsSerializer
import re
import time
import datetime, time
from . import models, fields, utils, utils_ldap, utils_auth, utils_stats
from .. import settings, secrets
@@ -229,6 +230,20 @@ class MemberSerializer(serializers.ModelSerializer):
logger.info(msg)
raise ValidationError(dict(discourse_username='Invalid Discourse username.'))
if validated_data.get('allow_last_scanned', None) == True:
changed = validated_data['allow_last_scanned'] != instance.allow_last_scanned
ONE_WEEK = now() - datetime.timedelta(days=7)
if changed and models.HistoryChange.objects.filter(
field='allow_last_scanned',
index__history_user__member__id=instance.id,
index__owner_id=instance.id,
index__history_date__gte=ONE_WEEK,
).count() >= 6:
msg = 'Member allow_last_scanned rate limit exceeded by: ' + instance.first_name + ' ' + instance.last_name
utils.alert_tanner(msg)
logger.info(msg)
raise ValidationError(dict(allow_last_scanned='You\'re doing that too often.'))
return super().update(instance, validated_data)
# admin viewing member details
@@ -384,6 +399,7 @@ class CardSerializer(serializers.ModelSerializer):
read_only_fields = [
'id',
'last_seen',
'last_seen_at',
'user',
]
@@ -536,7 +552,7 @@ class MyRegisterSerializer(RegisterSerializer):
if re.search(r'[^a-z.]', username):
raise ValidationError('Invalid characters.')
if '..' in username:
raise ValidationError('Can\'t have double periods.')
raise ValidationError('Can\'t have double periods. Remove spaces.')
if username.startswith('.') or username.endswith('.'):
raise ValidationError('Can\'t start or end with periods.')
return super().validate_username(username)

14
apiserver/apiserver/api/signals.py
View File

@@ -69,7 +69,7 @@ def post_create_historical_record_callback(
is_admin=is_admin_director(history_user),
)
for change in changes:
for num, change in enumerate(changes):
change_old = str(change.old)
change_new = str(change.new)
@@ -84,6 +84,18 @@ def post_create_historical_record_callback(
old=change_old,
new=change_new,
)
logger.info('History - {} changed {}\'s {} {}/{}: {} "{}" --> "{}"'.format(
history_user or 'System',
owner[0],
object_name,
num+1,
len(changes),
change.field,
change_old,
change_new,
))
except BaseException as e:
logger.error('History Signal - {} - {}'.format(e.__class__.__name__, e))
logger.info(str(sender))

35
apiserver/apiserver/api/throttles.py Normal file
View File

@@ -0,0 +1,35 @@
import logging
logger = logging.getLogger(__name__)
from rest_framework import throttling
class LoggingThrottle(throttling.BaseThrottle):
def allow_request(self, request, view):
if request.user.id:
user = '{} ({})'.format(request.user, request.user.member.id)
else:
user = None
method = request._request.method
path = request._request.path
if method == 'OPTIONS':
return True
if path.startswith('/lockout/'):
return True
elif path == '/stats/sign/':
pass
elif path.startswith('/stats/'):
return True
if request.data:
data = request.data.dict()
for key in ['password', 'password1', 'password2', 'old_password', 'new_password1', 'new_password2']:
if key in data:
data[key] = '[CENSORED]'
else:
data = None
logging.info('%s %s | User: %s | Data: %s', method, path, user, data)
return True

10
apiserver/apiserver/api/utils.py
View File

@@ -146,6 +146,8 @@ def gen_search_strings():
'''
Generate a cache dict of names to member ids for rapid string matching
'''
start = time.time()
search_strings = {}
for m in models.Member.objects.order_by('-expire_date'):
string = '{} {} | {} {}'.format(
@@ -162,6 +164,8 @@ def gen_search_strings():
search_strings[string] = m.id
cache.set('search_strings', search_strings)
logger.info('Generated search strings in %s s.', time.time() - start)
LARGE_SIZE = 1080
MEDIUM_SIZE = 220
@@ -317,9 +321,9 @@ def create_new_member(data, user):
models.Member.objects.create(
user=user,
first_name=data['first_name'].title(),
last_name=data['last_name'].title(),
preferred_name=data['first_name'].title(),
first_name=data['first_name'].title().strip(),
last_name=data['last_name'].title().strip(),
preferred_name=data['first_name'].title().strip(),
)
def register_user(data, user):

3
apiserver/apiserver/api/utils_stats.py
View File

@@ -26,6 +26,7 @@ DEFAULTS = {
'card_scans': 0,
'track': {},
'alarm': {},
'sign': '',
}
if secrets.MUMBLE:
@@ -158,7 +159,7 @@ def get_progress(request_id):
return cache.get('request-progress-' + request_id, [])
def set_progress(request_id, data):
logger.info('Request %s progress: %s', request_id, data)
logger.info('Progress - ID: %s | Status: %s', request_id, data)
progress = get_progress(request_id)
progress.append(data)
cache.set('request-progress-' + request_id, progress)

135
apiserver/apiserver/api/views.py
View File

@@ -4,7 +4,7 @@ logger = logging.getLogger(__name__)
from django.contrib.auth.models import User, Group
from django.shortcuts import get_object_or_404, redirect
from django.db import transaction
from django.db.models import Max, F
from django.db.models import Max, F, Count
from django.db.utils import OperationalError
from django.http import HttpResponse, Http404, FileResponse
from django.core.files.base import File
@@ -43,7 +43,7 @@ Create = mixins.CreateModelMixin
Update = mixins.UpdateModelMixin
Destroy = mixins.DestroyModelMixin
NUM_SEARCH_RESULTS = 20
NUM_RESULTS = 100
class SearchViewSet(Base, Retrieve):
@@ -79,54 +79,57 @@ class SearchViewSet(Base, Retrieve):
if len(results) == 0 and len(search) >= 3 and '@' not in search:
# then get fuzzy matches, but not for emails
fuzzy_results = process.extract(search, choices, limit=NUM_SEARCH_RESULTS, scorer=fuzz.token_set_ratio)
fuzzy_results = process.extract(search, choices, limit=20, scorer=fuzz.token_set_ratio)
results += [x[0] for x in fuzzy_results]
# remove dupes, truncate list
results = list(OrderedDict.fromkeys(results))[:NUM_SEARCH_RESULTS]
results = list(OrderedDict.fromkeys(results))[:20]
result_ids = [search_strings[x] for x in results]
result_objects = [queryset.get(id=x) for x in result_ids]
queryset = result_objects
logging.info('Search for: {}, results: {}'.format(search, len(queryset)))
elif self.action == 'create' and sort == 'recently_vetted':
utils.gen_search_strings() # update cache
queryset = queryset.order_by('-vetted_date')
elif self.action == 'create' and sort == 'newest_active':
queryset = queryset.filter(paused_date__isnull=True)
queryset = queryset.order_by('-application_date')
elif self.action == 'create' and sort == 'newest_overall':
queryset = queryset.order_by('-application_date')
elif self.action == 'create' and sort == 'oldest_active':
queryset = queryset.filter(paused_date__isnull=True)
queryset = queryset.order_by('application_date')
elif self.action == 'create' and sort == 'oldest_overall':
queryset = queryset.filter(application_date__isnull=False)
queryset = queryset.order_by('application_date')
elif self.action == 'create' and sort == 'recently_inactive':
queryset = queryset.filter(paused_date__isnull=False)
queryset = queryset.order_by('-paused_date')
elif self.action == 'create' and sort == 'is_director':
queryset = queryset.filter(is_director=True)
queryset = queryset.order_by('application_date')
elif self.action == 'create' and sort == 'is_instructor':
queryset = queryset.filter(is_instructor=True)
queryset = queryset.order_by('application_date')
elif self.action == 'create' and sort == 'due':
queryset = queryset.filter(status='Due')
queryset = queryset.order_by('expire_date')
elif self.action == 'create' and sort == 'overdue':
queryset = queryset.filter(status='Overdue')
queryset = queryset.order_by('expire_date')
elif self.action == 'create' and sort == 'last_scanned':
if self.request.user.member.allow_last_scanned:
queryset = queryset.filter(allow_last_scanned=True)
queryset = queryset.order_by('-user__cards__last_seen')
else:
elif self.action == 'create':
if sort == 'recently_vetted':
queryset = queryset.filter(vetted_date__isnull=False)
queryset = queryset.order_by('-vetted_date')
elif sort == 'newest_active':
queryset = queryset.filter(paused_date__isnull=True)
queryset = queryset.order_by('-application_date')
elif sort == 'newest_overall':
queryset = queryset.order_by('-application_date')
elif sort == 'oldest_active':
queryset = queryset.filter(paused_date__isnull=True)
queryset = queryset.order_by('application_date')
elif sort == 'oldest_overall':
queryset = queryset.filter(application_date__isnull=False)
queryset = queryset.order_by('application_date')
elif sort == 'recently_inactive':
queryset = queryset.filter(paused_date__isnull=False)
queryset = queryset.order_by('-paused_date')
elif sort == 'is_director':
queryset = queryset.filter(is_director=True)
queryset = queryset.order_by('application_date')
elif sort == 'is_instructor':
queryset = queryset.filter(is_instructor=True)
queryset = queryset.order_by('application_date')
elif sort == 'due':
queryset = queryset.filter(status='Due')
queryset = queryset.order_by('expire_date')
elif sort == 'overdue':
queryset = queryset.filter(status='Overdue')
queryset = queryset.order_by('expire_date')
elif sort == 'last_scanned':
if self.request.user.member.allow_last_scanned:
queryset = queryset.filter(allow_last_scanned=True)
queryset = queryset.order_by('-user__cards__last_seen')
else:
queryset = []
elif sort == 'everyone':
queryset = queryset.annotate(Count('user__transactions')).order_by('-user__transactions__count')
elif sort == 'best_looking':
queryset = []
elif self.action == 'create' and sort == 'best_looking':
queryset = []
return queryset
@@ -139,19 +142,19 @@ class SearchViewSet(Base, Retrieve):
seq = 0
search = self.request.data.get('q', '').lower()
if search:
num_results = NUM_SEARCH_RESULTS
else:
num_results = 100
page = self.request.data.get('page', 0)
queryset = self.get_queryset()
total = len(queryset)
queryset = self.get_queryset()[:num_results]
start = int(page) * NUM_RESULTS - 80
queryset = queryset[max(start,0):start+NUM_RESULTS]
if self.request.user.member.vetted_date:
serializer = serializers.VettedSearchSerializer(queryset, many=True)
else:
serializer = serializers.SearchSerializer(queryset, many=True)
return Response({'seq': seq, 'results': serializer.data})
return Response({'seq': seq, 'results': serializer.data, 'total': total})
class MemberViewSet(Base, Retrieve, Update):
@@ -168,11 +171,13 @@ class MemberViewSet(Base, Retrieve, Update):
member = serializer.save()
utils.tally_membership_months(member)
utils.gen_member_forms(member)
utils.gen_search_strings()
def perform_update(self, serializer):
member = serializer.save()
utils.tally_membership_months(member)
utils.gen_member_forms(member)
utils.gen_search_strings()
@action(detail=True, methods=['post'])
def pause(self, request, pk=None):
@@ -427,16 +432,13 @@ class PingView(views.APIView):
permission_classes = [AllowMetadata | IsAuthenticated]
def post(self, request):
d = request.data.dict()
if d:
logger.info(str(d))
return Response(200)
class DoorViewSet(viewsets.ViewSet, List):
def list(self, request):
auth_token = request.META.get('HTTP_AUTHORIZATION', '')
if auth_token != 'Bearer ' + secrets.DOOR_API_TOKEN:
if secrets.DOOR_API_TOKEN and auth_token != 'Bearer ' + secrets.DOOR_API_TOKEN:
raise exceptions.PermissionDenied()
cards = models.Card.objects.filter(active_status='card_active')
@@ -462,7 +464,7 @@ class DoorViewSet(viewsets.ViewSet, List):
member = card.user.member
t = utils.now_alberta_tz().strftime('%Y-%m-%d %H:%M:%S, %a %I:%M %p')
logger.info('Time: {} - Name: {} {} ({})'.format(t, member.first_name, member.last_name, member.id))
logger.info('Scan - Time: {} | Name: {} {} ({})'.format(t, member.first_name, member.last_name, member.id))
utils_stats.calc_card_scans()
@@ -472,7 +474,7 @@ class DoorViewSet(viewsets.ViewSet, List):
class LockoutViewSet(viewsets.ViewSet, List):
def list(self, request):
auth_token = request.META.get('HTTP_AUTHORIZATION', '')
if auth_token != 'Bearer ' + secrets.DOOR_API_TOKEN:
if secrets.DOOR_API_TOKEN and auth_token != 'Bearer ' + secrets.DOOR_API_TOKEN:
raise exceptions.PermissionDenied()
cards = models.Card.objects.filter(active_status='card_active')
@@ -553,6 +555,24 @@ class StatsViewSet(viewsets.ViewSet, List):
except KeyError:
raise exceptions.ValidationError(dict(data='This field is required.'))
@action(detail=False, methods=['post'])
def sign(self, request):
try:
sign = request.data['sign']
cache.set('sign', sign)
if secrets.SIGN_TOKEN:
try:
post_data = dict(access_token=secrets.SIGN_TOKEN, args=sign)
r = requests.post('https://api.particle.io/v1/devices/200042000647343232363230/text/', data=post_data, timeout=5)
r.raise_for_status()
except:
raise exceptions.ValidationError(dict(sign='Something went wrong :('))
return Response(200)
except KeyError:
raise exceptions.ValidationError(dict(sign='This field is required.'))
@action(detail=False, methods=['post'])
def alarm(self, request):
try:
@@ -635,7 +655,7 @@ class BackupView(views.APIView):
backup_user = secrets.BACKUP_TOKENS.get(auth_token, None)
if backup_user:
logger.info('Backup user: ' + backup_user['name'])
logger.info('Backup - User: ' + backup_user['name'])
backup_path = cache.get(backup_user['cache_key'], None)
if not backup_path:
@@ -720,15 +740,6 @@ class VettingViewSet(Base, List):
class RegistrationView(RegisterView):
serializer_class = serializers.MyRegisterSerializer
def post(self, request):
data = request.data.copy()
data.pop('password1', None)
data.pop('password2', None)
logger.info(dict(data))
return super().post(request)
class PasswordChangeView(PasswordChangeView):
permission_classes = [AllowMetadata | IsAuthenticated]
serializer_class = serializers.MyPasswordChangeSerializer

3
apiserver/apiserver/secrets.py.example
View File

@@ -1,4 +1,6 @@
# Spaceport secrets file, don't commit to version control!
#
# Note: all values are optional, features are excluded if left blank
# /admin/ route obfuscation
# Set this to random characters
@@ -59,6 +61,7 @@ DOOR_CODE = ''
WIFI_PASS = ''
MINECRAFT = ''
MUMBLE = ''
SIGN_TOKEN = ''
# Portal Email Credentials
# For sending password resets, etc.

6
apiserver/apiserver/settings.py
View File

@@ -25,7 +25,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = secrets.DJANGO_SECRET_KEY
SECRET_KEY = secrets.DJANGO_SECRET_KEY or 'OaOBN2E+brpoRyDMlTD9eTE5PgBtkkl+L7Bzt6pQ5Qr3GS82SH'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG_ENV = os.environ.get('DEBUG', False)
@@ -55,7 +55,6 @@ SESSION_COOKIE_SECURE = True
CSRF_COOKIE_SECURE = True
SECURE_REFERRER_POLICY = 'same-origin'
# Application definition
INSTALLED_APPS = [
@@ -209,6 +208,7 @@ REST_FRAMEWORK = {
'PAGE_SIZE': 300,
'DEFAULT_RENDERER_CLASSES': DEFAULT_RENDERER_CLASSES,
'DEFAULT_AUTHENTICATION_CLASSES': DEFAULT_AUTHENTICATION_CLASSES,
'DEFAULT_THROTTLE_CLASSES': ['apiserver.api.throttles.LoggingThrottle'],
}
#DEFAULT_LOGGING = None
@@ -239,7 +239,7 @@ LOGGING = {
'loggers': {
'gunicorn': {
'handlers': ['console'],
'level': 'DEBUG',
'level': 'DEBUG' if DEBUG else 'INFO',
'propagate': False,
},
'': {

13
apiserver/apiserver/urls.py
View File

@@ -7,9 +7,6 @@ from rest_auth.views import LoginView, LogoutView
from .api import views
from . import secrets, settings
IPN_ROUTE = r'^ipn/{}/'.format(secrets.IPN_RANDOM)
ADMIN_ROUTE = '{}/admin/'.format(secrets.ADMIN_RANDOM)
router = routers.DefaultRouter()
router.register(r'door', views.DoorViewSet, basename='door')
router.register(r'lockout', views.LockoutViewSet, basename='lockout')
@@ -31,7 +28,6 @@ router.register(r'charts/spaceactivity', views.SpaceActivityViewSet, basename='s
urlpatterns = [
path('', include(router.urls)),
path(ADMIN_ROUTE, admin.site.urls),
url(r'^rest-auth/login/$', LoginView.as_view(), name='rest_login'),
url(r'^spaceport-auth/login/$', views.SpaceportAuthView.as_view(), name='spaceport_auth'),
url(r'^rest-auth/logout/$', LogoutView.as_view(), name='rest_logout'),
@@ -44,9 +40,16 @@ urlpatterns = [
url(r'^ping/', views.PingView.as_view(), name='ping'),
url(r'^paste/', views.PasteView.as_view(), name='paste'),
url(r'^backup/', views.BackupView.as_view(), name='backup'),
url(IPN_ROUTE, views.IpnView.as_view(), name='ipn'),
]
if secrets.IPN_RANDOM:
IPN_ROUTE = r'^ipn/{}/'.format(secrets.IPN_RANDOM)
urlpatterns.append(url(IPN_ROUTE, views.IpnView.as_view(), name='ipn'))
if secrets.ADMIN_RANDOM:
ADMIN_ROUTE = '{}/admin/'.format(secrets.ADMIN_RANDOM)
urlpatterns.append(path(ADMIN_ROUTE, admin.site.urls))
if settings.DEBUG:
urlpatterns += [
path('api-auth/', include('rest_framework.urls')),

6
apiserver/requirements.txt
View File

@@ -1,11 +1,11 @@
alabaster==0.7.12
argon2-cffi==19.2.0
asgiref==3.2.3
Babel==2.9.1
bleach==3.3.0
certifi==2019.11.28
cffi==1.13.2
chardet==3.0.4
commonmark==0.9.1
defusedxml==0.6.0
Django==3.1.13
django-allauth==0.41.0
@@ -18,6 +18,7 @@ gunicorn==20.0.4
idna==2.8
imagesize==1.2.0
Jinja2==2.11.3
logging-tree==1.8.1
MarkupSafe==1.1.1
oauthlib==3.1.0
packaging==20.0
@@ -31,6 +32,7 @@ python-Levenshtein==0.12.0
python-memcached==1.59
python3-openid==3.1.0
pytz==2019.3
recommonmark==0.7.1
reportlab==3.5.34
requests==2.22.0
requests-oauthlib==1.3.0
@@ -46,5 +48,5 @@ sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.2
sphinxcontrib-serializinghtml==1.1.3
sqlparse==0.3.0
urllib3==1.26.5
urllib3==1.25.11
webencodings==0.5.1