# Petale - Simple App as Key/Value Storage Interface # Copyright (C) 2017 Entr'ouvert # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU Affero General Public License as published # by the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from __future__ import unicode_literals import logging try: from functools import reduce except ImportError: pass import requests from django.utils.six.moves.urllib import parse as urlparse from django.db.models.query import Q, F from django.http import StreamingHttpResponse, HttpResponse from django.conf import settings from django.db.transaction import atomic from rest_framework import status from rest_framework.views import APIView from rest_framework.response import Response from .models import CUT, Petal, Partner, AccessControlList from .utils import logit, StreamingHash from .exceptions import (PartnerNotFound, CutNotFound, KeyNotFound, NotFound, MissingContentType, ConcurrentAccess, PreconditionException) def cut_exists(request, cut_uuid): logger = logging.getLogger(__name__) if not getattr(settings, 'PETALE_CHECK_CUT_UUID', True): CUT.objects.get_or_create(uuid=cut_uuid) return True authentic_url = getattr(settings, 'PETALE_AUTHENTIC_URL', None) if not authentic_url: logger.warning(u'PETALE_AUTHENTIC SETTINGS improperly defined') return False url = urlparse.urljoin(authentic_url, 'api/users/synchronization/') try: response = requests.post(url, json={"known_uuids": [cut_uuid]}, auth=request.user.credentials, verify=False) response.raise_for_status() except requests.RequestException as e: logger.warning(u'authentic synchro API failed: %s', e) return False try: data = response.json() except ValueError as e: logger.warning(u'authentic synchro API failed to decode response: %s', e) return False if data.get("unknown_uuids"): logger.warning(u'unknown uuid : %s %s', request.user.credentials[0], data) return False CUT.objects.get_or_create(uuid=cut_uuid) return True class PetalAPIKeysView(APIView): http_method_names = ['get', 'delete'] @logit def get(self, request, partner_name, cut_uuid): try: partner = Partner.objects.get(name=partner_name) except Partner.DoesNotExist: raise PartnerNotFound qs = Petal.objects.filter(partner=partner, cut_id__uuid=cut_uuid) if not qs.exists(): if not CUT.objects.filter(uuid=cut_uuid).exists(): if not cut_exists(request, cut_uuid): raise CutNotFound key_filter = None # filter by param if request.query_params.get('prefix'): key_filter = Q(name__startswith=request.query_params['prefix']) # filter by acls acls = AccessControlList.objects.all() acls = acls.filter(partner=partner, user=request.user) prefixes = [] for acl in acls: if acl.key == '*': prefixes = [] break prefixes.append(acl.key.strip('*')) if prefixes: acls_filter = reduce(Q.__or__, (Q(name__startswith=prefix) for prefix in prefixes)) if key_filter: key_filter &= acls_filter else: key_filter = acls_filter if key_filter: qs = qs.filter(key_filter) return Response({ 'keys': [petal.name for petal in qs] }) @logit def delete(self, request, partner_name, cut_uuid): try: partner = Partner.objects.get(name=partner_name) except Partner.DoesNotExist: raise PartnerNotFound Petal.objects.filter(partner=partner, cut_id__uuid=cut_uuid).delete() return Response(status=status.HTTP_204_NO_CONTENT) class PetalAPIView(APIView): http_method_names = ['get', 'head', 'put', 'delete'] def get_petal(self, partner_name, cut_uuid, petal_name, if_match=None, if_none_match=None): if_match = if_match and [x.strip() for x in if_match.split(',')] if_none_match = if_none_match and [x.strip() for x in if_none_match.split(',')] try: qs = Petal.objects.filter( name=petal_name, partner_id__name=partner_name, cut_id__uuid=cut_uuid ) qs = qs.select_related('partner', 'cut') petal = qs.get() except Petal.DoesNotExist: if if_match: raise PreconditionException # Only check cut uuid on IDP during creation if not CUT.objects.filter(uuid=cut_uuid).exists(): if not cut_exists(self.request, cut_uuid): raise CutNotFound raise KeyNotFound if if_match: if if_match == ['*']: pass elif petal.etag not in if_match: raise PreconditionException if if_none_match: if if_none_match == ['*']: raise PreconditionException if petal.etag in if_none_match: raise PreconditionException return petal @logit def head(self, request, partner_name, cut_uuid, petal_name): petal = self.get_petal(partner_name, cut_uuid, petal_name) response = HttpResponse(content_type=petal.content_type) response['Content-Length'] = petal.size response['ETag'] = petal.etag return response @logit def get(self, request, partner_name, cut_uuid, petal_name): logger = logging.getLogger(__name__) if_none_match = request.META.get('HTTP_IF_NONE_MATCH') if_none_match = if_none_match and [x.strip() for x in if_none_match.split(',')] petal = self.get_petal(partner_name, cut_uuid, petal_name) if if_none_match: if if_none_match == ['*'] or petal.etag in if_none_match: return Response( status=status.HTTP_304_NOT_MODIFIED, headers={ 'ETag': petal.etag } ) # verify file exists before creating a StreamingHttpResponse # as StreamingHttpResponse generate its content after the Django global try/catch try: petal.data.open() except IOError as e: logger.error('file not found "%s": %s', petal.data.path, e) return HttpResponse('missing file', status=500) response = StreamingHttpResponse( petal.data.chunks(), content_type=petal.content_type, ) response['ETag'] = petal.etag return response @logit @atomic def put(self, request, partner_name, cut_uuid, petal_name): # pylint: disable=too-many-locals if_match = request.META.get('HTTP_IF_MATCH') if_none_match = request.META.get('HTTP_IF_NONE_MATCH') content_type = request.META.get('CONTENT_TYPE') # check that content-type is set if not content_type: raise MissingContentType try: petal = self.get_petal(partner_name, cut_uuid, petal_name, if_match=if_match, if_none_match=if_none_match) created = False except PreconditionException: raise ConcurrentAccess except NotFound: try: partner = Partner.objects.get(name=partner_name) except Partner.DoesNotExist: raise PartnerNotFound try: cut = CUT.objects.get(uuid=cut_uuid) except CUT.DoesNotExist: raise CutNotFound petal, created = Petal.objects.get_or_create( name=petal_name, partner=partner, cut=cut, defaults={'size': 0}) if not created and if_none_match: raise ConcurrentAccess else: if if_none_match: raise ConcurrentAccess status_code = status.HTTP_200_OK try: content_length = int(request.META.get('CONTENT_LENGTH')) except (ValueError, TypeError): content_length = 0 size_delta = content_length - petal.size petal.check_limits(content_length) if created: status_code = status.HTTP_201_CREATED old_name = None else: old_name = petal.data.name streaming_digest = StreamingHash(request) petal.data.save(petal_name, streaming_digest, save=False) # update metadata petal.content_type = content_type petal.etag = streaming_digest.etag() petal.size = content_length # update partner size Partner.objects.filter(id=petal.partner.id).update(size=F('size') + size_delta) petal.save() if old_name: petal.data.storage.delete(old_name) return Response( {}, status=status_code, headers={ 'ETag': petal.etag }) @logit def delete(self, request, partner_name, cut_uuid, petal_name): if_match = request.META.get('HTTP_IF_MATCH') if_none_match = request.META.get('HTTP_IF_NONE_MATCH') try: petal = self.get_petal( partner_name, cut_uuid, petal_name, if_match=if_match, if_none_match=if_none_match) except PreconditionException: raise ConcurrentAccess petal.delete() return Response(status=status.HTTP_204_NO_CONTENT)