From 1ea429bd2608924d66ad1b589ea584fb0dc14a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Wed, 10 Aug 2022 13:53:47 +0200 Subject: [PATCH] general: remove unused document types & validation code (#14148) --- fargo/fargo/admin.py | 7 -- fargo/fargo/api_views.py | 79 +------------------ .../migrations/0004_auto_20220810_1352.py | 18 +++++ fargo/fargo/models.py | 56 ------------- fargo/fargo/utils.py | 6 -- fargo/fargo/views.py | 13 --- fargo/settings.py | 22 ------ fargo/urls.py | 5 +- tests/test_api.py | 40 ---------- 9 files changed, 21 insertions(+), 225 deletions(-) create mode 100644 fargo/fargo/migrations/0004_auto_20220810_1352.py diff --git a/fargo/fargo/admin.py b/fargo/fargo/admin.py index a5d8ccf..16cb9de 100644 --- a/fargo/fargo/admin.py +++ b/fargo/fargo/admin.py @@ -88,12 +88,6 @@ class DocumentAdmin(admin.ModelAdmin): thumbnail.short_description = _('thumbnail') -class ValidationAdmin(admin.ModelAdmin): - fields = ['user', 'content_hash', 'document_type', 'start', 'end', 'data', 'creator', 'created', 'origin'] - readonly_fields = ['created'] - list_display = ['user', 'display', 'document_type', 'start', 'end', 'creator', 'created', 'origin'] - - class OriginAdmin(admin.ModelAdmin): fields = ['label', 'slug'] list_display = ['label', 'slug'] @@ -101,5 +95,4 @@ class OriginAdmin(admin.ModelAdmin): admin.site.register(models.UserDocument, UserDocumentAdmin) admin.site.register(models.Document, DocumentAdmin) -admin.site.register(models.Validation, ValidationAdmin) admin.site.register(models.Origin, OriginAdmin) diff --git a/fargo/fargo/api_views.py b/fargo/fargo/api_views.py index babd31f..2b7abcd 100644 --- a/fargo/fargo/api_views.py +++ b/fargo/fargo/api_views.py @@ -18,16 +18,14 @@ import datetime from django.conf import settings from django.contrib.auth.models import User -from django.urls import reverse from django.utils.text import slugify -from django.utils.timezone import now -from rest_framework import exceptions, filters, mixins, routers, serializers, status, viewsets +from rest_framework import exceptions, filters, serializers, status from rest_framework.generics import GenericAPIView, ListAPIView from rest_framework.permissions import IsAdminUser from rest_framework.response import Response from . import api_errors, api_fields, utils -from .models import Document, Origin, UserDocument, Validation +from .models import Document, Origin, UserDocument try: from mellon.models import UserSAMLIdentifier @@ -166,46 +164,6 @@ class RecentDocuments(ListAPIView): recent_documents = RecentDocuments.as_view() -class ValidationSerializer(UserSerializerMixin, serializers.ModelSerializer): - origin = api_fields.SlugCreatedRelatedField(slug_field='label', queryset=Origin.objects.all()) - url = serializers.SerializerMethodField() - display = serializers.CharField(read_only=True) - - def __init__(self, *args, **kwargs): - schema = kwargs.pop('schema') - super().__init__(*args, **kwargs) - self.document_type = schema['name'] - self.document_type_schema = schema - for field in schema['metadata']: - name = field['varname'] - required = field.get('required', True) - self.fields[name] = serializers.CharField( - source='data.%s' % name, required=required, allow_blank=True - ) - - def get_url(self, instance): - url = reverse( - 'fargo-api-validation-detail', kwargs={'document_type': instance.document_type, 'pk': instance.pk} - ) - if 'request' in self.context: - url = self.context['request'].build_absolute_uri(url) - return url - - def validate(self, data): - data = super().validate(data) - data['document_type'] = self.document_type - data['created'] = now().replace(microsecond=0) - data['start'] = data['created'].date() - data['end'] = data['start'] + datetime.timedelta(seconds=settings.FARGO_VALIDATION_LIFETIME) - data['creator'] = data['creator'][:256] - return data - - class Meta: - model = Validation - exclude = ('data', 'user', 'document_type') - read_only_fields = ('created', 'start', 'end') - - class FilterByUser(filters.BaseFilterBackend): def filter_queryset(self, request, queryset, view): if 'user_email' in request.GET: @@ -213,36 +171,3 @@ class FilterByUser(filters.BaseFilterBackend): elif 'user_nameid' in request.GET: return queryset.filter(user__saml_identifiers__name_id=request.GET['user_nameid']) return queryset - - -class ValidationAPI( - CommonAPIMixin, - mixins.CreateModelMixin, - mixins.RetrieveModelMixin, - mixins.ListModelMixin, - viewsets.GenericViewSet, -): - serializer_class = ValidationSerializer - permission_classes = (IsAdminUser,) - filter_backends = [FilterByUser] - queryset = Validation.objects.all() - - def get_queryset(self): - return super().get_queryset().filter(document_type=self.document_type) - - def initial(self, request, document_type, *args, **kwargs): - self.document_type_schema = utils.get_document_type_schema(settings, document_type) - if not self.document_type_schema: - error = serializers.ValidationError('unknown document type') - error.status_code = status.HTTP_404_NOT_FOUND - raise error - self.document_type = document_type - super().initial(request, document_type, *args, **kwargs) - - def get_serializer(self, *args, **kwargs): - # pass schema to serializer class - return super().get_serializer(schema=self.document_type_schema, *args, **kwargs) - - -router = routers.SimpleRouter() -router.register(r'validation/(?P[^/]*)', ValidationAPI, basename='fargo-api-validation') diff --git a/fargo/fargo/migrations/0004_auto_20220810_1352.py b/fargo/fargo/migrations/0004_auto_20220810_1352.py new file mode 100644 index 0000000..d494a55 --- /dev/null +++ b/fargo/fargo/migrations/0004_auto_20220810_1352.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2022-08-10 11:52 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('fargo', '0003_text_to_jsonb'), + ] + + operations = [ + migrations.DeleteModel( + name='Validation', + ), + ] diff --git a/fargo/fargo/models.py b/fargo/fargo/models.py index 4f31bb7..030a969 100644 --- a/fargo/fargo/models.py +++ b/fargo/fargo/models.py @@ -115,62 +115,6 @@ class UserDocument(models.Model): ) -@python_2_unicode_compatible -class Validation(models.Model): - """Validation of a document as special kind for an user, - the data field contains metadata extracted from the document. - """ - - user = models.ForeignKey(settings.AUTH_USER_MODEL, verbose_name=_('user'), on_delete=models.CASCADE) - content_hash = models.CharField(max_length=128, verbose_name=_('content hash'), blank=True, null=True) - origin = models.ForeignKey(Origin, verbose_name=_('origin'), null=True, on_delete=models.CASCADE) - document_type = models.CharField(max_length=256, verbose_name=_('document type')) - data = JSONField(null=True, verbose_name=_('data'), default=dict) - start = models.DateField(verbose_name=_('start date')) - end = models.DateField(verbose_name=_('end date')) - creator = models.CharField(max_length=256, verbose_name=_('creator')) - created = models.DateTimeField(verbose_name=_('creation date')) - - @property - def document_type_schema(self): - return utils.get_document_type_schema(settings, self.document_type) or {} - - @property - def metadata(self): - return self.document_type_schema.get('metadata', []) - - def display(self): - template = self.document_type_schema.get('display_template', '') - if template: - try: - return force_text(template.format(**self.data)) - except KeyError: - pass - parts = [] - for meta_field in self.metadata: - parts.append( - _('%(label)s: %(value)s') - % { - 'label': meta_field['label'], - 'value': self.data.get(meta_field['varname'], ''), - } - ) - return force_text('; '.join(parts)) - - display.short_description = _('description') - - @property - def user_document(self): - if self.content_hash: - try: - return UserDocument.objects.get(document__content_hash=self.content_hash) - except UserDocument.DoesNotExist: - pass - - def __str__(self): - return self.display() - - class Document(models.Model): '''Content indexed documents''' diff --git a/fargo/fargo/utils.py b/fargo/fargo/utils.py index dadf3fc..b96a142 100644 --- a/fargo/fargo/utils.py +++ b/fargo/fargo/utils.py @@ -30,12 +30,6 @@ def to_isodate(dt): return dt.astimezone(utc).isoformat('T').split('.')[0] + 'Z' -def get_document_type_schema(settings, document_type): - for schema in settings.FARGO_DOCUMENT_TYPES: - if schema.get('name') == document_type: - return schema - - def sha256_of_file(f): '''Compute SHA-256 hash of Django File object''' hasher = hashlib.sha256() diff --git a/fargo/fargo/views.py b/fargo/fargo/views.py index 90c8473..a0c7ed3 100644 --- a/fargo/fargo/views.py +++ b/fargo/fargo/views.py @@ -329,18 +329,6 @@ class LogoutView(auth_views.LogoutView): logout = LogoutView.as_view() -class DocumentTypes(View): - def get(self, request): - document_types = deepcopy(settings.FARGO_DOCUMENT_TYPES) - for document_type in document_types: - document_type.pop('display_template', None) - data = { - 'err': 0, - 'data': document_types, - } - return HttpResponse(dumps(data), content_type='application/json') - - home = login_required(Homepage.as_view()) download = login_required(Download.as_view()) thumbnail = login_required(Thumbnail.as_view()) @@ -352,4 +340,3 @@ pick = login_required(Pick.as_view()) jsonp = login_required(JSONP.as_view()) json = login_required(JSON.as_view()) pick_list = xframe_options_exempt(login_required(PickList.as_view())) -document_types = DocumentTypes.as_view() diff --git a/fargo/settings.py b/fargo/settings.py index a2430ed..4d07060 100644 --- a/fargo/settings.py +++ b/fargo/settings.py @@ -180,28 +180,6 @@ FARGO_MAX_DOCUMENTS_PER_USER = 100 FARGO_VALIDATION_LIFETIME = 3600 * 24 * 31 * 6 # nearly 6 months -FARGO_DOCUMENT_TYPES = [ - { - 'name': 'avis-d-imposition', - 'label': 'Avis d\'imposition', - 'metadata': [ - {'label': 'Personne-s concernée-s', 'varname': 'personnes_concernees', 'type': 'string'}, - { - 'label': 'Année', - 'varname': 'annee', - 'type': 'string', - 'validation': ' *[0-9]{4} *', - }, - { - 'label': 'Revenu fiscal de référence', - 'varname': 'revenu_fiscal_de_reference', - 'type': 'string', - 'validation': ' *[0-9]+ *', - }, - ], - }, -] - LOGIN_REDIRECT_URL = 'home' REST_FRAMEWORK = { diff --git a/fargo/urls.py b/fargo/urls.py index 60bbd89..1d008d5 100644 --- a/fargo/urls.py +++ b/fargo/urls.py @@ -18,10 +18,9 @@ from django.conf import settings from django.conf.urls import include, url from django.contrib import admin -from .fargo.api_views import push_document, recent_documents, router +from .fargo.api_views import push_document, recent_documents from .fargo.views import ( delete, - document_types, download, edit, home, @@ -51,10 +50,8 @@ urlpatterns = [ url(r'^admin/', admin.site.urls), url(r'^login/$', login, name='auth_login'), url(r'^logout/$', logout, name='auth_logout'), - url(r'^document-types/$', document_types, name='document_types'), url(r'^api/documents/push/$', push_document, name='fargo-api-push-document'), url(r'^api/documents/recently-added/$', recent_documents), - url(r'^api/', include(router.urls)), ] if settings.DEBUG and 'debug_toolbar' in settings.INSTALLED_APPS: diff --git a/tests/test_api.py b/tests/test_api.py index 0935c88..62ad548 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -25,46 +25,6 @@ from fargo.fargo import models, utils pytestmark = pytest.mark.django_db -def test_create_validation(settings, app, admin_user, john_doe, jane_doe): - login(app) - data = { - 'user_email': john_doe.email, - } - schema = utils.get_document_type_schema(settings, 'avis-d-imposition') - url = '/api/validation/avis-d-imposition/' - assert models.Validation.objects.count() == 0 - response = app.post_json(url, params=data, status=400) - assert response.json['result'] == 0 - assert set(response.json['errors'].keys()) == set( - [field['varname'] for field in schema['metadata']] + ['creator', 'origin'] - ) - assert models.Validation.objects.count() == 0 - data.update( - { - 'personnes_concernees': 'John and Lisa Doe', - 'annee': '2016', - 'revenu_fiscal_de_reference': '32455', - 'creator': 'FooBar', - 'origin': 'wcs.example.com', - } - ) - response1 = app.post_json(url, params=data, status=201) - assert set(response1.json.keys()) == {'result', 'data'} - assert response1.json['result'] == 1 - assert set(data.keys()) - {'user_email'} < set(response1.json['data'].keys()) - assert models.Validation.objects.count() == 1 - data['user_email'] = jane_doe.email - response2 = app.post_json(url, params=data, status=201) - assert models.Validation.objects.count() == 2 - response3 = app.get(url) - assert response3.json['data']['count'] == 2 - response4 = app.get(url + '?%s' % urlencode({'user_email': john_doe.email})) - assert response4.json['data']['count'] == 1 - response5 = app.get(response2.json['data']['url'], status=200) - assert response5.json['result'] == 1 - assert response5.json['data'] == response2.json['data'] - - def test_push_document(app, admin_user, john_doe): login(app) data = {