general: remove unused document types & validation code (#14148)

fargo/fargo/admin.py

@@ -88,12 +88,6 @@ class DocumentAdmin(admin.ModelAdmin):
     thumbnail.short_description = _('thumbnail')
 
 
-class ValidationAdmin(admin.ModelAdmin):
-    fields = ['user', 'content_hash', 'document_type', 'start', 'end', 'data', 'creator', 'created', 'origin']
-    readonly_fields = ['created']
-    list_display = ['user', 'display', 'document_type', 'start', 'end', 'creator', 'created', 'origin']
-
-
 class OriginAdmin(admin.ModelAdmin):
     fields = ['label', 'slug']
     list_display = ['label', 'slug']
@@ -101,5 +95,4 @@ class OriginAdmin(admin.ModelAdmin):
 
 admin.site.register(models.UserDocument, UserDocumentAdmin)
 admin.site.register(models.Document, DocumentAdmin)
-admin.site.register(models.Validation, ValidationAdmin)
 admin.site.register(models.Origin, OriginAdmin)

fargo/fargo/api_views.py

@@ -18,16 +18,14 @@ import datetime
 
 from django.conf import settings
 from django.contrib.auth.models import User
-from django.urls import reverse
 from django.utils.text import slugify
-from django.utils.timezone import now
+from rest_framework import exceptions, filters, serializers, status
-from rest_framework import exceptions, filters, mixins, routers, serializers, status, viewsets
 from rest_framework.generics import GenericAPIView, ListAPIView
 from rest_framework.permissions import IsAdminUser
 from rest_framework.response import Response
 
 from . import api_errors, api_fields, utils
-from .models import Document, Origin, UserDocument, Validation
+from .models import Document, Origin, UserDocument
 
 try:
     from mellon.models import UserSAMLIdentifier
@@ -166,46 +164,6 @@ class RecentDocuments(ListAPIView):
 recent_documents = RecentDocuments.as_view()
 
 
-class ValidationSerializer(UserSerializerMixin, serializers.ModelSerializer):
-    origin = api_fields.SlugCreatedRelatedField(slug_field='label', queryset=Origin.objects.all())
-    url = serializers.SerializerMethodField()
-    display = serializers.CharField(read_only=True)
-
-    def __init__(self, *args, **kwargs):
-        schema = kwargs.pop('schema')
-        super().__init__(*args, **kwargs)
-        self.document_type = schema['name']
-        self.document_type_schema = schema
-        for field in schema['metadata']:
-            name = field['varname']
-            required = field.get('required', True)
-            self.fields[name] = serializers.CharField(
-                source='data.%s' % name, required=required, allow_blank=True
-            )
-
-    def get_url(self, instance):
-        url = reverse(
-            'fargo-api-validation-detail', kwargs={'document_type': instance.document_type, 'pk': instance.pk}
-        )
-        if 'request' in self.context:
-            url = self.context['request'].build_absolute_uri(url)
-        return url
-
-    def validate(self, data):
-        data = super().validate(data)
-        data['document_type'] = self.document_type
-        data['created'] = now().replace(microsecond=0)
-        data['start'] = data['created'].date()
-        data['end'] = data['start'] + datetime.timedelta(seconds=settings.FARGO_VALIDATION_LIFETIME)
-        data['creator'] = data['creator'][:256]
-        return data
-
-    class Meta:
-        model = Validation
-        exclude = ('data', 'user', 'document_type')
-        read_only_fields = ('created', 'start', 'end')
-
-
 class FilterByUser(filters.BaseFilterBackend):
     def filter_queryset(self, request, queryset, view):
         if 'user_email' in request.GET:
@@ -213,36 +171,3 @@ class FilterByUser(filters.BaseFilterBackend):
         elif 'user_nameid' in request.GET:
             return queryset.filter(user__saml_identifiers__name_id=request.GET['user_nameid'])
         return queryset
-
-
-class ValidationAPI(
-    CommonAPIMixin,
-    mixins.CreateModelMixin,
-    mixins.RetrieveModelMixin,
-    mixins.ListModelMixin,
-    viewsets.GenericViewSet,
-):
-    serializer_class = ValidationSerializer
-    permission_classes = (IsAdminUser,)
-    filter_backends = [FilterByUser]
-    queryset = Validation.objects.all()
-
-    def get_queryset(self):
-        return super().get_queryset().filter(document_type=self.document_type)
-
-    def initial(self, request, document_type, *args, **kwargs):
-        self.document_type_schema = utils.get_document_type_schema(settings, document_type)
-        if not self.document_type_schema:
-            error = serializers.ValidationError('unknown document type')
-            error.status_code = status.HTTP_404_NOT_FOUND
-            raise error
-        self.document_type = document_type
-        super().initial(request, document_type, *args, **kwargs)
-
-    def get_serializer(self, *args, **kwargs):
-        # pass schema to serializer class
-        return super().get_serializer(schema=self.document_type_schema, *args, **kwargs)
-
-
-router = routers.SimpleRouter()
-router.register(r'validation/(?P<document_type>[^/]*)', ValidationAPI, basename='fargo-api-validation')

fargo/fargo/migrations/0004_auto_20220810_1352.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.28 on 2022-08-10 11:52
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('fargo', '0003_text_to_jsonb'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='Validation',
+        ),
+    ]

fargo/fargo/models.py

@@ -115,62 +115,6 @@ class UserDocument(models.Model):
         )
 
 
-@python_2_unicode_compatible
-class Validation(models.Model):
-    """Validation of a document as special kind for an user,
-    the data field contains metadata extracted from the document.
-    """
-
-    user = models.ForeignKey(settings.AUTH_USER_MODEL, verbose_name=_('user'), on_delete=models.CASCADE)
-    content_hash = models.CharField(max_length=128, verbose_name=_('content hash'), blank=True, null=True)
-    origin = models.ForeignKey(Origin, verbose_name=_('origin'), null=True, on_delete=models.CASCADE)
-    document_type = models.CharField(max_length=256, verbose_name=_('document type'))
-    data = JSONField(null=True, verbose_name=_('data'), default=dict)
-    start = models.DateField(verbose_name=_('start date'))
-    end = models.DateField(verbose_name=_('end date'))
-    creator = models.CharField(max_length=256, verbose_name=_('creator'))
-    created = models.DateTimeField(verbose_name=_('creation date'))
-
-    @property
-    def document_type_schema(self):
-        return utils.get_document_type_schema(settings, self.document_type) or {}
-
-    @property
-    def metadata(self):
-        return self.document_type_schema.get('metadata', [])
-
-    def display(self):
-        template = self.document_type_schema.get('display_template', '')
-        if template:
-            try:
-                return force_text(template.format(**self.data))
-            except KeyError:
-                pass
-        parts = []
-        for meta_field in self.metadata:
-            parts.append(
-                _('%(label)s: %(value)s')
-                % {
-                    'label': meta_field['label'],
-                    'value': self.data.get(meta_field['varname'], ''),
-                }
-            )
-        return force_text('; '.join(parts))
-
-    display.short_description = _('description')
-
-    @property
-    def user_document(self):
-        if self.content_hash:
-            try:
-                return UserDocument.objects.get(document__content_hash=self.content_hash)
-            except UserDocument.DoesNotExist:
-                pass
-
-    def __str__(self):
-        return self.display()
-
-
 class Document(models.Model):
     '''Content indexed documents'''
 

fargo/fargo/utils.py

@@ -30,12 +30,6 @@ def to_isodate(dt):
     return dt.astimezone(utc).isoformat('T').split('.')[0] + 'Z'
 
 
-def get_document_type_schema(settings, document_type):
-    for schema in settings.FARGO_DOCUMENT_TYPES:
-        if schema.get('name') == document_type:
-            return schema
-
-
 def sha256_of_file(f):
     '''Compute SHA-256 hash of Django File object'''
     hasher = hashlib.sha256()

fargo/fargo/views.py

@@ -329,18 +329,6 @@ class LogoutView(auth_views.LogoutView):
 logout = LogoutView.as_view()
 
 
-class DocumentTypes(View):
-    def get(self, request):
-        document_types = deepcopy(settings.FARGO_DOCUMENT_TYPES)
-        for document_type in document_types:
-            document_type.pop('display_template', None)
-        data = {
-            'err': 0,
-            'data': document_types,
-        }
-        return HttpResponse(dumps(data), content_type='application/json')
-
-
 home = login_required(Homepage.as_view())
 download = login_required(Download.as_view())
 thumbnail = login_required(Thumbnail.as_view())
@@ -352,4 +340,3 @@ pick = login_required(Pick.as_view())
 jsonp = login_required(JSONP.as_view())
 json = login_required(JSON.as_view())
 pick_list = xframe_options_exempt(login_required(PickList.as_view()))
-document_types = DocumentTypes.as_view()

fargo/settings.py

@@ -180,28 +180,6 @@ FARGO_MAX_DOCUMENTS_PER_USER = 100
 
 FARGO_VALIDATION_LIFETIME = 3600 * 24 * 31 * 6  # nearly 6 months
 
-FARGO_DOCUMENT_TYPES = [
-    {
-        'name': 'avis-d-imposition',
-        'label': 'Avis d\'imposition',
-        'metadata': [
-            {'label': 'Personne-s concernée-s', 'varname': 'personnes_concernees', 'type': 'string'},
-            {
-                'label': 'Année',
-                'varname': 'annee',
-                'type': 'string',
-                'validation': ' *[0-9]{4} *',
-            },
-            {
-                'label': 'Revenu fiscal de référence',
-                'varname': 'revenu_fiscal_de_reference',
-                'type': 'string',
-                'validation': ' *[0-9]+ *',
-            },
-        ],
-    },
-]
-
 LOGIN_REDIRECT_URL = 'home'
 
 REST_FRAMEWORK = {

fargo/urls.py

@@ -18,10 +18,9 @@ from django.conf import settings
 from django.conf.urls import include, url
 from django.contrib import admin
 
-from .fargo.api_views import push_document, recent_documents, router
+from .fargo.api_views import push_document, recent_documents
 from .fargo.views import (
     delete,
-    document_types,
     download,
     edit,
     home,
@@ -51,10 +50,8 @@ urlpatterns = [
     url(r'^admin/', admin.site.urls),
     url(r'^login/$', login, name='auth_login'),
     url(r'^logout/$', logout, name='auth_logout'),
-    url(r'^document-types/$', document_types, name='document_types'),
     url(r'^api/documents/push/$', push_document, name='fargo-api-push-document'),
     url(r'^api/documents/recently-added/$', recent_documents),
-    url(r'^api/', include(router.urls)),
 ]
 
 if settings.DEBUG and 'debug_toolbar' in settings.INSTALLED_APPS:

tests/test_api.py

@@ -25,46 +25,6 @@ from fargo.fargo import models, utils
 pytestmark = pytest.mark.django_db
 
 
-def test_create_validation(settings, app, admin_user, john_doe, jane_doe):
-    login(app)
-    data = {
-        'user_email': john_doe.email,
-    }
-    schema = utils.get_document_type_schema(settings, 'avis-d-imposition')
-    url = '/api/validation/avis-d-imposition/'
-    assert models.Validation.objects.count() == 0
-    response = app.post_json(url, params=data, status=400)
-    assert response.json['result'] == 0
-    assert set(response.json['errors'].keys()) == set(
-        [field['varname'] for field in schema['metadata']] + ['creator', 'origin']
-    )
-    assert models.Validation.objects.count() == 0
-    data.update(
-        {
-            'personnes_concernees': 'John and Lisa Doe',
-            'annee': '2016',
-            'revenu_fiscal_de_reference': '32455',
-            'creator': 'FooBar',
-            'origin': 'wcs.example.com',
-        }
-    )
-    response1 = app.post_json(url, params=data, status=201)
-    assert set(response1.json.keys()) == {'result', 'data'}
-    assert response1.json['result'] == 1
-    assert set(data.keys()) - {'user_email'} < set(response1.json['data'].keys())
-    assert models.Validation.objects.count() == 1
-    data['user_email'] = jane_doe.email
-    response2 = app.post_json(url, params=data, status=201)
-    assert models.Validation.objects.count() == 2
-    response3 = app.get(url)
-    assert response3.json['data']['count'] == 2
-    response4 = app.get(url + '?%s' % urlencode({'user_email': john_doe.email}))
-    assert response4.json['data']['count'] == 1
-    response5 = app.get(response2.json['data']['url'], status=200)
-    assert response5.json['result'] == 1
-    assert response5.json['data'] == response2.json['data']
-
-
 def test_push_document(app, admin_user, john_doe):
     login(app)
     data = {