313 lines
11 KiB
Python
313 lines
11 KiB
Python
# Petale - Simple App as Key/Value Storage Interface
|
|
# Copyright (C) 2017 Entr'ouvert
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify it
|
|
# under the terms of the GNU Affero General Public License as published
|
|
# by the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
import logging
|
|
from urllib import parse as urlparse
|
|
|
|
try:
|
|
from functools import reduce
|
|
except ImportError:
|
|
pass
|
|
import requests
|
|
|
|
try:
|
|
from time import process_time
|
|
except ImportError:
|
|
from time import clock as process_time
|
|
|
|
from atomicwrites import atomic_write
|
|
from django.conf import settings
|
|
from django.db.models.query import F, Q
|
|
from django.db.transaction import atomic
|
|
from django.http import HttpResponse
|
|
from rest_framework import status
|
|
from rest_framework.response import Response
|
|
from rest_framework.views import APIView
|
|
|
|
from .exceptions import (
|
|
ConcurrentAccess,
|
|
CutNotFound,
|
|
KeyNotFound,
|
|
MissingContentType,
|
|
NotFound,
|
|
PartnerNotFound,
|
|
PreconditionException,
|
|
)
|
|
from .models import CUT, AccessControlList, Partner, Petal
|
|
from .utils import StreamingHash, logit
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
SENTINEL = object()
|
|
|
|
|
|
def check_unknown_cuts(uuids, creds=SENTINEL):
|
|
authentic_url = getattr(settings, 'PETALE_AUTHENTIC_URL', None)
|
|
if not authentic_url:
|
|
raise ValueError('PETALE_AUTHENTIC SETTINGS improperly defined')
|
|
|
|
authentic_creds = creds if creds is not SENTINEL else getattr(settings, 'PETALE_AUTHENTIC_CREDS', None)
|
|
if not authentic_creds:
|
|
raise ValueError('missing credentials for authentic, configure PETALE_AUTHENTIC_CREDS')
|
|
url = urlparse.urljoin(authentic_url, 'api/users/synchronization/')
|
|
response = requests.post(url, json={"known_uuids": list(uuids)}, auth=authentic_creds, verify=False)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
return data.get("unknown_uuids") or []
|
|
|
|
|
|
def cut_exists(request, cut_uuid):
|
|
if not getattr(settings, 'PETALE_CHECK_CUT_UUID', True):
|
|
CUT.objects.get_or_create(uuid=cut_uuid)
|
|
return True
|
|
|
|
authentic_url = getattr(settings, 'PETALE_AUTHENTIC_URL', None)
|
|
if not authentic_url:
|
|
logger.warning('PETALE_AUTHENTIC SETTINGS improperly defined')
|
|
return False
|
|
|
|
url = urlparse.urljoin(authentic_url, 'api/users/synchronization/')
|
|
try:
|
|
response = requests.post(
|
|
url, json={"known_uuids": [cut_uuid]}, auth=request.user.credentials, verify=False
|
|
)
|
|
response.raise_for_status()
|
|
except requests.RequestException as e:
|
|
logger.warning('authentic synchro API failed: %s', e)
|
|
return False
|
|
try:
|
|
data = response.json()
|
|
except ValueError as e:
|
|
logger.warning('authentic synchro API failed to decode response: %s', e)
|
|
return False
|
|
|
|
if data.get("unknown_uuids"):
|
|
logger.warning('unknown uuid : %s %s', request.user.credentials[0], data)
|
|
return False
|
|
|
|
CUT.objects.get_or_create(uuid=cut_uuid)
|
|
return True
|
|
|
|
|
|
class PetalAPIKeysView(APIView):
|
|
http_method_names = ['get', 'delete']
|
|
|
|
@logit
|
|
def get(self, request, partner_name, cut_uuid):
|
|
try:
|
|
partner = Partner.objects.get(name=partner_name)
|
|
except Partner.DoesNotExist:
|
|
raise PartnerNotFound
|
|
|
|
qs = Petal.objects.filter(partner=partner, cut_id__uuid=cut_uuid)
|
|
|
|
if not qs.exists():
|
|
if not CUT.objects.filter(uuid=cut_uuid).exists():
|
|
if not cut_exists(request, cut_uuid):
|
|
raise CutNotFound
|
|
|
|
key_filter = None
|
|
|
|
# filter by param
|
|
if request.query_params.get('prefix'):
|
|
key_filter = Q(name__startswith=request.query_params['prefix'])
|
|
|
|
# filter by acls
|
|
acls = AccessControlList.objects.all()
|
|
acls = acls.filter(partner=partner, user=request.user)
|
|
prefixes = []
|
|
for acl in acls:
|
|
if acl.key == '*':
|
|
prefixes = []
|
|
break
|
|
prefixes.append(acl.key.strip('*'))
|
|
if prefixes:
|
|
acls_filter = reduce(Q.__or__, (Q(name__startswith=prefix) for prefix in prefixes))
|
|
if key_filter:
|
|
key_filter &= acls_filter
|
|
else:
|
|
key_filter = acls_filter
|
|
|
|
if key_filter:
|
|
qs = qs.filter(key_filter)
|
|
|
|
return Response({'keys': [petal.name for petal in qs]})
|
|
|
|
@logit
|
|
def delete(self, request, partner_name, cut_uuid):
|
|
try:
|
|
partner = Partner.objects.get(name=partner_name)
|
|
except Partner.DoesNotExist:
|
|
raise PartnerNotFound
|
|
|
|
Petal.objects.filter(partner=partner, cut_id__uuid=cut_uuid).delete()
|
|
|
|
return Response(status=status.HTTP_204_NO_CONTENT)
|
|
|
|
|
|
class PetalAPIView(APIView):
|
|
http_method_names = ['get', 'head', 'put', 'delete']
|
|
|
|
def get_petal(self, partner_name, cut_uuid, petal_name, if_match=None, if_none_match=None):
|
|
if_match = if_match and [x.strip() for x in if_match.split(',')]
|
|
if_none_match = if_none_match and [x.strip() for x in if_none_match.split(',')]
|
|
|
|
try:
|
|
qs = Petal.objects.filter(name=petal_name, partner_id__name=partner_name, cut_id__uuid=cut_uuid)
|
|
qs = qs.select_related('partner', 'cut')
|
|
petal = qs.get()
|
|
except Petal.DoesNotExist:
|
|
if if_match:
|
|
raise PreconditionException
|
|
# Only check cut uuid on IDP during creation
|
|
if not CUT.objects.filter(uuid=cut_uuid).exists():
|
|
if not cut_exists(self.request, cut_uuid):
|
|
raise CutNotFound
|
|
raise KeyNotFound
|
|
|
|
if if_match:
|
|
if if_match == ['*']:
|
|
pass
|
|
elif petal.etag not in if_match:
|
|
raise PreconditionException
|
|
|
|
if if_none_match:
|
|
if if_none_match == ['*']:
|
|
raise PreconditionException
|
|
if petal.etag in if_none_match:
|
|
raise PreconditionException
|
|
return petal
|
|
|
|
@logit
|
|
def head(self, request, partner_name, cut_uuid, petal_name):
|
|
petal = self.get_petal(partner_name, cut_uuid, petal_name)
|
|
response = HttpResponse(content_type=petal.content_type)
|
|
response['Content-Length'] = petal.size
|
|
response['ETag'] = petal.etag
|
|
return response
|
|
|
|
@logit
|
|
def get(self, request, partner_name, cut_uuid, petal_name):
|
|
if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
|
|
if_none_match = if_none_match and [x.strip() for x in if_none_match.split(',')]
|
|
|
|
start = process_time()
|
|
while process_time() - start < 10:
|
|
petal = self.get_petal(partner_name, cut_uuid, petal_name)
|
|
if if_none_match:
|
|
if if_none_match == ['*'] or petal.etag in if_none_match:
|
|
return Response(status=status.HTTP_304_NOT_MODIFIED, headers={'ETag': petal.etag})
|
|
# verify file exists before creating a StreamingHttpResponse
|
|
# as StreamingHttpResponse generate its content after the Django global try/catch
|
|
try:
|
|
petal.data.open(mode='rb')
|
|
response = HttpResponse(petal.data.read(), content_type=petal.content_type)
|
|
except OSError:
|
|
continue
|
|
finally:
|
|
petal.data.close()
|
|
break
|
|
else:
|
|
return HttpResponse(status=500, reason='Too many updates')
|
|
response['ETag'] = petal.etag
|
|
return response
|
|
|
|
@logit
|
|
@atomic
|
|
def put(self, request, partner_name, cut_uuid, petal_name):
|
|
# pylint: disable=too-many-locals
|
|
if_match = request.META.get('HTTP_IF_MATCH')
|
|
if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
|
|
content_type = request.META.get('CONTENT_TYPE')
|
|
|
|
# check that content-type is set
|
|
if not content_type:
|
|
raise MissingContentType
|
|
|
|
try:
|
|
petal = self.get_petal(
|
|
partner_name, cut_uuid, petal_name, if_match=if_match, if_none_match=if_none_match
|
|
)
|
|
created = False
|
|
except PreconditionException:
|
|
raise ConcurrentAccess
|
|
except NotFound:
|
|
try:
|
|
partner = Partner.objects.get(name=partner_name)
|
|
except Partner.DoesNotExist:
|
|
raise PartnerNotFound
|
|
try:
|
|
cut = CUT.objects.get(uuid=cut_uuid)
|
|
except CUT.DoesNotExist:
|
|
raise CutNotFound
|
|
|
|
petal, created = Petal.objects.get_or_create(
|
|
name=petal_name, partner=partner, cut=cut, defaults={'size': 0}
|
|
)
|
|
if not created and if_none_match:
|
|
raise ConcurrentAccess
|
|
else:
|
|
if if_none_match:
|
|
raise ConcurrentAccess
|
|
|
|
status_code = status.HTTP_200_OK
|
|
try:
|
|
content_length = int(request.META.get('CONTENT_LENGTH'))
|
|
except (ValueError, TypeError):
|
|
content_length = 0
|
|
size_delta = content_length - petal.size
|
|
|
|
petal.check_limits(content_length)
|
|
|
|
streaming_digest = StreamingHash(request)
|
|
|
|
def update_meta():
|
|
# update metadata
|
|
petal.content_type = content_type
|
|
petal.etag = streaming_digest.etag()
|
|
petal.size = content_length
|
|
|
|
# update partner size
|
|
Partner.objects.filter(id=petal.partner.id).update(size=F('size') + size_delta)
|
|
petal.save()
|
|
|
|
if created:
|
|
status_code = status.HTTP_201_CREATED
|
|
petal.data.save(petal_name, streaming_digest, save=False)
|
|
update_meta()
|
|
else:
|
|
with atomic_write(petal.data.path, overwrite=True, mode='wb') as fd:
|
|
for block in iter(lambda: streaming_digest.read(512), b''):
|
|
fd.write(block)
|
|
update_meta()
|
|
|
|
return Response({}, status=status_code, headers={'ETag': petal.etag})
|
|
|
|
@logit
|
|
def delete(self, request, partner_name, cut_uuid, petal_name):
|
|
if_match = request.META.get('HTTP_IF_MATCH')
|
|
if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
|
|
|
|
try:
|
|
petal = self.get_petal(
|
|
partner_name, cut_uuid, petal_name, if_match=if_match, if_none_match=if_none_match
|
|
)
|
|
except PreconditionException:
|
|
raise ConcurrentAccess
|
|
petal.delete()
|
|
return Response(status=status.HTTP_204_NO_CONTENT)
|