api: export content of uploads (#7254)

help/fr/api-get.page

@@ -71,6 +71,10 @@ Le contenu ainsi obtenu est le suivant :
 }
 </code>
 
+<p>
+Seuls les champs ayant un <em>nom de variable</em> sont exportés dans <code>fields</code>.
+</p>
+
 <note>
  <p>
   Il est bien sûr nécessaire de disposer des autorisations nécessaires pour
@@ -102,6 +106,47 @@ l'exemple ci-dessus).
 <section id="datatypes">
 <title>Types de données</title>
 
+<p>
+Les données d'un formulaire sont placées dans le champs <code>fields</code> de
+la réponse. Les champs de type simple tels que « Texte », « Texte long » ou
+« Courriel » sont vus en tant que chaîne de caractères :
+</p>
+
+  <code mime="application/json">
+    (...)
+    "fields": {
+        "email": "marc@example.net",
+        "nom": "L.",
+        "prenom": "Marc"
+    },
+    (...)
+  </code>
+
+<section>
+<title>Représentation d'un champ « Fichier »</title>
+
+<p>
+Les champs de type « Fichier » sont exportés selon le schéma suivant :
+</p>
+
+  <code mime="application/json">
+    (...)
+    "fields": {
+        "photo": {
+            "filename": "exemple.txt",
+            "content_type": "text/plain",
+            "content": "Q2VjaSBuJ2VzdCBwYXMgdW4gZXhlbXBsZS4="
+        }
+    },
+    (...)
+  </code>
+
+<p>
+La valeur de <code>content</code> est le contenu du fichier, encodé en base64.
+</p>
+
+</section>
+
 </section>
 
 <section id="listing">
@@ -184,6 +229,11 @@ l'adresse.
        https://www.example.net/api/forms/inscriptions/list?full=on</input>
 </screen>
 
+<p>
+À noter que pour ne pas alourdir l'export en mode <code>full=on</code>, les
+champs de type « Fichier » ne sont pas exportés.
+</p>
+
 </section>
 
 

tests/test_api.py

@@ -10,6 +10,7 @@ import datetime
 import time
 
 from quixote import cleanup, get_publisher
+from wcs.qommon.form import PicklableUpload
 from wcs.users import User
 from wcs.roles import Role
 from wcs.formdef import FormDef
@@ -242,22 +243,28 @@ def test_formdata(local_user):
     formdef.fields = [
         fields.StringField(id='0', label='foobar', varname='foobar'),
         fields.StringField(id='1', label='foobar2'),
-        fields.DateField(id='2', label='foobar3', varname='date'),]
+        fields.DateField(id='2', label='foobar3', varname='date'),
+        fields.FileField(id='3', label='foobar4', varname='file'),]
     formdef.store()
 
     formdata = formdef.data_class()()
     date = time.strptime('2014-01-20', '%Y-%m-%d')
-    formdata.data = {'0': 'foo@localhost', '1': 'xxx', '2': date}
+    upload = PicklableUpload('test.txt', 'text/plain', 'ascii')
+    upload.receive(['base64me'])
+    formdata.data = {'0': 'foo@localhost', '1': 'xxx', '2': date, '3': upload}
     formdata.user_id = local_user.id
     formdata.just_created()
     formdata.store()
 
     resp = get_app(pub).get(sign_uri('/test/%s/' % formdata.id, user=local_user))
     assert 'last_update_time' in resp.json
+    assert len(resp.json['fields']) == 3 # foobar2 has no varname, not in json
     assert resp.json['user']['name'] == local_user.name
     assert resp.json['fields']['foobar'] == 'foo@localhost'
     assert resp.json['fields']['date'] == '2014-01-20'
-    assert len(resp.json['fields']) == 2 # foobar2 has no varname, not in json
+    assert resp.json['fields']['file']['content'] == 'YmFzZTY0bWU=' # base64('base64me')
+    assert resp.json['fields']['file']['filename'] == 'test.txt'
+    assert resp.json['fields']['file']['content_type'] == 'text/plain'
 
 def test_myspace_forms(local_user):
     FormDef.wipe()
@@ -318,6 +325,7 @@ def test_api_list_formdata(local_user):
         fields.StringField(id='0', label='foobar', varname='foobar'),
         fields.ItemField(id='1', label='foobar3', varname='foobar3', type='item',
             items=['foo', 'bar', 'baz']),
+        fields.FileField(id='2', label='foobar4', varname='file'),
         ]
     formdef.store()
 
@@ -327,7 +335,9 @@ def test_api_list_formdata(local_user):
     for i in range(30):
         formdata = data_class()
         date = time.strptime('2014-01-20', '%Y-%m-%d')
-        formdata.data = {'0': 'FOO BAR %d' % i}
+        upload = PicklableUpload('test.txt', 'text/plain', 'ascii')
+        upload.receive(['base64me'])
+        formdata.data = {'0': 'FOO BAR %d' % i, '2': upload}
         if i%4 == 0:
             formdata.data['1'] = 'foo'
             formdata.data['1_display'] = 'foo'
@@ -363,6 +373,7 @@ def test_api_list_formdata(local_user):
     assert len(resp.json) == 30
     assert 'receipt_time' in resp.json[0]
     assert 'fields' in resp.json[0]
+    assert 'file' not in resp.json[0]['fields'] # no file export in full lists
 
     # check filtered results
     resp = get_app(pub).get(sign_uri('/api/forms/test/list?filter-foobar3=foo', user=local_user))

wcs/backoffice/management.py

@@ -861,7 +861,7 @@ class FormPage(Directory):
             selected_filter, user=user, query=query, criterias=criterias,
             order_by=order_by)
         if get_request().form.get('full') == 'on':
-            output = [json.loads(filled.export_to_json()) for filled in items]
+            output = [json.loads(filled.export_to_json(include_files=False)) for filled in items]
         else:
             output = [{'id': filled.id,
                 'url': filled.get_url(),

wcs/fields.py

@@ -17,6 +17,7 @@
 import time
 import random
 import re
+import base64
 import xml.etree.ElementTree as ET
 
 from quixote import get_request, get_publisher
@@ -709,7 +710,12 @@ class FileField(WidgetField):
         return ['%s' % value]
 
     def get_json_value(self, value):
-        return {'field_id': self.id, 'filename': value.base_filename}
+        return {
+            'field_id': self.id,
+            'filename': value.base_filename,
+            'content_type': value.content_type or 'application/octet-stream',
+            'content': base64.b64encode(value.get_content())
+        }
 
     def perform_more_widget_changes(self, form, kwargs, edit = True):
         if not edit:

wcs/formdata.py

@@ -28,6 +28,7 @@ import qommon.misc
 from qommon.substitution import Substitutions
 
 from roles import Role
+from fields import FileField
 
 
 def get_dict_with_varnames(fields, data, formdata=None, varnames_only=False):
@@ -87,11 +88,13 @@ def flatten_dict(d):
             del d[k]
 
 
-def get_json_dict(fields, data):
+def get_json_dict(fields, data, include_files=True):
     new_data = {}
     for field in fields:
         if not field.varname: # exports only named fields
             continue
+        if not include_files and isinstance(field, FileField):
+            continue
         if data is not None:
             value = data.get(field.id)
             if value and hasattr(field, 'get_json_value'):
@@ -527,7 +530,7 @@ class FormData(StorableObject):
                 evo.parts = None
         self.store()
 
-    def export_to_json(self):
+    def export_to_json(self, include_files=True):
         data = {}
         data['id'] = '%s/%s' % (self.formdef.url_name, self.id)
         data['display_id'] = self.get_display_id()
@@ -553,7 +556,8 @@ class FormData(StorableObject):
         if user:
             data['user'] = {'id': user.id, 'name': user.display_name}
 
-        data['fields'] = get_json_dict(self.formdef.fields, self.data)
+        data['fields'] = get_json_dict(self.formdef.fields, self.data,
+                include_files=include_files)
 
         data['workflow'] = {}
         wf_status = self.get_visible_status()

wcs/qommon/form.py

@@ -687,6 +687,12 @@ class PicklableUpload(Upload):
         # quack like UploadedFile
         return self.get_file_pointer()
 
+    def get_content(self):
+        if hasattr(self, 'qfilename'):
+            filename = os.path.join(get_publisher().app_dir, 'uploads', self.qfilename)
+            return file(filename).read()
+        return None
+
 
 class EmailWidget(StringWidget):
     HTML_TYPE = 'email'