Added HTTP Basic authentication, more logging, cleaning, some small fix

Feature:
    * added two new connector fields to store HTTP Basic credentials
    * removed one connector field that stored the HTTP auth token

Logging:
    * added debuging and warning messages
    * replaced the python interpolation '%' by function args ','

Cleaning:
    * removed functions 'ajob', 'afile'
    * removed unused imports

Fixes:
    * added missing imports
    * using string in dictionary keys
This commit is contained in:
Michael Bideau 2019-07-10 17:56:20 +02:00
parent 3c53170e4a
commit 58608886aa
2 changed files with 125 additions and 97 deletions

View File

@ -1,9 +1,8 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.15 on 2019-04-30 12:07
# Generated by Django 1.11.15 on 2019-07-10 14:51
from __future__ import unicode_literals
from django.db import migrations, models
import jsonfield.fields
import passerelle.apps.atreal_openads.models
@ -12,7 +11,7 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
('base', '0012_job'),
('base', '0013_delete_templatevar'),
]
operations = [
@ -23,11 +22,11 @@ class Migration(migrations.Migration):
('title', models.CharField(max_length=50, verbose_name='Title')),
('description', models.TextField(verbose_name='Description')),
('slug', models.SlugField(unique=True, verbose_name='Identifier')),
('myjobs', jsonfield.fields.JSONField(default={})),
('collectivite', models.CharField(blank=True, default=b'', help_text='ex: Marseille, ou ex: 3', max_length=255, verbose_name='Collectivity (identifier)')),
('openADS_API_key', models.CharField(default=b'', help_text='ex: ah9pGbKKHv5ToF3cPQuV', max_length=255, verbose_name='openADS API key (secret)')),
('collectivite', models.CharField(blank=True, default=b'', help_text='ex: Marseille, or ex: 3', max_length=255, verbose_name='Collectivity (identifier)')),
('openADS_API_login', models.CharField(default=b'', help_text='ex: user1234', max_length=255, verbose_name='openADS API login')),
('openADS_API_password', models.CharField(default=b'', help_text='ex: ah9pGbKKHv5ToF3cPQuV', max_length=255, verbose_name='openADS API password')),
('openADS_API_url', models.URLField(default=b'', help_text='ex: https://openads.your_domain.net/api/', max_length=255, verbose_name='openADS API URL')),
('users', models.ManyToManyField(blank=True, to='base.ApiUser')),
('users', models.ManyToManyField(blank=True, related_name='_atrealopenads_users_+', related_query_name='+', to='base.ApiUser')),
],
options={
'verbose_name': 'openADS',

View File

@ -20,30 +20,27 @@
import json
import base64
import urlparse
import datetime
import os
import re
import magic
import hashlib
import copy
import requests
import six
from HTMLParser import HTMLParser
from django.db import models
from django.http import Http404, HttpResponse, FileResponse
from django.http import Http404
from django.utils.translation import ugettext_lazy as _
from django.core.files import File
from django.core.files.base import ContentFile
from passerelle.base.models import BaseResource
from passerelle.utils.api import endpoint
from passerelle.utils.jsonresponse import APIError
import jsonfield
import os
import re
from HTMLParser import HTMLParser
import magic
import hashlib
from django.core.files import File
from django.core.files.base import ContentFile
# TODO remove (only for debuging/development)
import time
import copy
class MLStripper(HTMLParser):
"""HTML parser that removes html tags."""
@ -96,8 +93,13 @@ def get_file_data(path, b64=True):
def get_upload_path(instance, filename):
"""Return a relative upload path for a file."""
#return 'pass_openADS_up_%s' % instance.file_hash
return 'pass_openADS_up'
# be careful:
# * openADS accept only filename less than 50 chars
# * name should be unique, even if the content is the same
return 'pass_openADS_up_%s_%s' % (
datetime.datetime.now().strftime('%Y-%b-%d_%Hh%Mm%Ss%f'),
instance.file_hash[:4]
)
class ForwardFile(models.Model):
@ -116,10 +118,11 @@ class ForwardFile(models.Model):
class AtrealOpenads(BaseResource):
"""API that proxy/relay communications with/to openADS."""
myjobs = jsonfield.JSONField(default={})
collectivite = models.CharField(_('Collectivity (identifier)'), max_length=255,
help_text=_('ex: Marseille, or ex: 3'), default='', blank=True)
openADS_API_key = models.CharField(_('openADS API key (secret)'), max_length=255,
openADS_API_login = models.CharField(_('openADS API login'), max_length=255,
help_text=_('ex: user1234'), default='')
openADS_API_password = models.CharField(_('openADS API password'), max_length=255,
help_text=_('ex: ah9pGbKKHv5ToF3cPQuV'), default='')
openADS_API_url = models.URLField(_('openADS API URL'), max_length=255,
help_text=_('ex: https://openads.your_domain.net/api/'), default='')
@ -170,7 +173,13 @@ class AtrealOpenads(BaseResource):
if (up_files and len(up_files)) or empty:
req_infos[k] = []
for f in up_files:
req_infos[k].append({name: f.name, size: f.size, content_type: f.content_type, charset: f.charset, temporary_file_path: f.temporary_file_path()})
req_infos[k].append({
'name': f.name,
'size': f.size,
'content_type': f.content_type,
'charset': f.charset,
'temporary_file_path': f.temporary_file_path()
})
elif (k != 'COOKIES' or cookies) and (k != 'META' or meta):
it = getattr(request, k).items()
if (it and len(it)) or empty:
@ -186,14 +195,14 @@ class AtrealOpenads(BaseResource):
description="[DEV] Return the file it has received encoded in base64",
methods=['post'])
def echofile(self, request, *args, **kwargs):
self.logger.debug("echofile() request.content_type = '%s'" % request.content_type)
self.logger.debug("echofile() len(request.body) = '%d', type(request.body) = '%s'" % (len(request.body), request.body.__class__))
self.logger.debug("echofile() request.content_type = '%s'", request.content_type)
self.logger.debug("echofile() len(request.body) = '%d', type(request.body) = '%s'", len(request.body), request.body.__class__)
if request.content_type == 'application/json' and len(request.body):
json_data = json.loads(request.body)
self.logger.debug("echofile() 'url' in json = '%s'" % str('url' in json_data))
self.logger.debug("echofile() 'url' in json = '%s'", str('url' in json_data))
if 'url' in json_data:
url = json_data['url']
self.logger.debug("echofile() url = '%s'" % url)
self.logger.debug("echofile() url = '%s'", url)
try:
response = self.requests.get(url)
@ -205,9 +214,9 @@ class AtrealOpenads(BaseResource):
'error': six.text_type(e)
})
self.logger.debug("echofile() response is '%s' (%s)" % (response, response.__class__))
self.logger.debug("echofile() response is '%s' (%s)", response, response.__class__)
self.logger.debug("echofile() response.status_code = '%s'" % response.status_code)
self.logger.debug("echofile() response.status_code = '%s'", response.status_code)
if response.status_code != 200:
raise APIError(
'API-WCS returned a non 200 status %s: %s' % response.status_code,
@ -217,10 +226,10 @@ class AtrealOpenads(BaseResource):
})
if 'content-type' in response.headers:
self.logger.debug("echofile() response['content-type'] = '%s'" % response.headers['content-type'])
self.logger.debug("echofile() response['content-type'] = '%s'", response.headers['content-type'])
if 'content-disposition' in response.headers:
self.logger.debug("echofile() response['content-disposition'] = '%s'" % response.headers['content-disposition'])
self.logger.debug("echofile() response.content[:50] = '%s'" % response.content[:50])
self.logger.debug("echofile() response['content-disposition'] = '%s'", response.headers['content-disposition'])
self.logger.debug("echofile() response.content[:50] = '%s'", response.content[:50])
return {
'content_type' : response.content_type if hasattr(response, 'content_type') else 'application/octet-stream',
'content' : base64.b64encode(response.content)
@ -239,57 +248,16 @@ class AtrealOpenads(BaseResource):
'content_type' : content_type,
'content' : content
}
raise ValueError("invalid request payload (no 'url' or 'b64_content' key found)")
raise ValueError("invalid content type of request '%s' (but must be '%s')" % (request.content_type, 'application/json'))
@endpoint(
description="[DEV] Return a file structure with its content in base64 from an hardcoded file",
pattern='^(?P<format>\w+)/?$',
example_pattern='{format}/',
parameters={
'format': {'description': _('Format'), 'example_value': 'base64'}
})
def afile(self, request, format='json', **kwargs):
rand_id = base64.urlsafe_b64encode(os.urandom(6))
self.add_job('ajob', natural_id=rand_id, dossier_id=rand_id)
if format == 'base64':
return get_file_data(TEST_FILE_TRAC_ICO)
elif format[:4] == 'json':
json = {
'afile': {
'filename' : 'trac.ico',
'content_type': 'image/x-icon',
'b64_content' : get_file_data(TEST_FILE_TRAC_ICO)
},
'extra_info': 'blabla'
}
return json
else:
raw_content = get_file_data(TEST_FILE_TRAC_ICO, b64=False)
return HttpResponse(raw_content, content_type='image/x-icon')
#return FileResponse(raw_content, content_type='image/x-icon')
def ajob(self, dossier_id, *args, **kwargs):
"""A test job."""
self.myjobs[dossier_id] = 'started'
self.save()
self.logger.debug("Started ajob() %s" % dossier_id)
for i in range(10):
self.logger.debug("Updated ajob() %s" % dossier_id)
self.myjobs[dossier_id] = 'running'
self.save()
time.sleep(10)
self.logger.debug("Ended ajob() %s" % dossier_id)
self.myjobs[dossier_id] = 'ended'
self.save()
def check_status(self):
"""Check avaibility of the openADS.API service."""
url = urlparse.urljoin(self.openADS_API_url, '__api__')
response = self.requests.get(url)
response = self.requests.get(url, auth=(self.openADS_API_login, self.openADS_API_password))
response.raise_for_status()
return {'response': response.status_code}
@ -381,7 +349,7 @@ class AtrealOpenads(BaseResource):
if 'content_type' in json_data['fields'][k]:
content_type = json_data['fields'][k]['content_type']
if k == 'cerfa' and content_type != 'application/pdf':
self.logger.warning("CERFA content type is '%s' instead of '%s'" % (content_type, 'application/pdf'))
self.logger.warning("CERFA content type is '%s' instead of '%s'", content_type, 'application/pdf')
if 'filename' in json_data['fields'][k]:
filename = json_data['fields'][k]['filename']
files.append({
@ -397,26 +365,44 @@ class AtrealOpenads(BaseResource):
self.logger.debug("----- files (end) -----")
url = urlparse.urljoin(self.openADS_API_url, '/dossiers/%s' % type_dossier)
response = self.requests.post(url, json=payload)
response = self.requests.post(url, json=payload, auth=(self.openADS_API_login, self.openADS_API_password))
if response.status_code // 100 != 2:
error = self.get_response_error(response)
self.logger.warning("Request [POST] '%s' failed with error: '%s'" % (url, error))
self.logger.warning("Request [POST] '%s' failed with error: '%s'", url, error)
raise APIError(error)
try:
result = response.json()
except ValueError:
raise APIError('No JSON content returned: %r' % response.content[:1000])
self.logger.debug("----- response json (begining) -----")
debug_json = copy.deepcopy(result)
if 'files' in debug_json \
and debug_json['files'] \
and isinstance(debug_json['files'], list) \
and len(debug_json['files']) > 0 \
and isinstance(debug_json['files'][0], dict) \
and 'b64_content' in debug_json['files'][0]:
debug_json['files'][0]['b64_content'] = '<b64 content>'
self.logger.debug(json.dumps(debug_json))
self.logger.debug("----- response json (end) -----")
numero_dossier = result.get('numero_dossier')
self.logger.debug("Numéro dossier: %s", str(numero_dossier))
recepisse = result['files'][0]
try:
recepisse_content = base64.b64decode(recepisse['b64_content'])
except TypeError:
raise APIError('Invalid content for recepisse')
self.logger.debug("Successfully decoded recepisse from base64")
if recepisse['content_type'] and recepisse['content_type'] != 'application/pdf':
self.logger.warning("Forcing 'recepisse' content type to '%s' instead of '%s'." % ('application/pdf', recepisse['content_type']))
self.logger.warning(
"Forcing 'recepisse' content type to '%s' instead of '%s'.",
'application/pdf',
recepisse['content_type']
)
recepisse['content_type'] = 'application/pdf'
if files:
@ -431,13 +417,26 @@ class AtrealOpenads(BaseResource):
FF.upload_file.save(FF.orig_filename, f['upload_file'])
FF.upload_status = 'pending'
FF.save()
self.logger.debug(
"Created ForwardFile '%s' for file '%s' (%s)",
FF.id,
FF.orig_filename,
FF.upload_file.path
)
file_ids.append(FF.id)
self.add_job('upload_user_files',
job = self.add_job('upload_user_files',
natural_id=numero_dossier,
type_dossier=type_dossier,
numero_dossier=numero_dossier,
file_ids=file_ids)
self.logger.debug(
"Added a job '%s' for dossier '%s' (%s) with file ids '%s'",
job.id,
numero_dossier,
type_dossier,
','.join([str(fid) for fid in file_ids])
)
return {
'numero_dossier': numero_dossier,
@ -475,10 +474,10 @@ class AtrealOpenads(BaseResource):
})
def get_dossier(self, request, type_dossier, numero_dossier, *args, **kwargs):
url = urlparse.urljoin(self.openADS_API_url, '/dossier/%s/%s' % (type_dossier, numero_dossier))
response = self.requests.get(url)
response = self.requests.get(url, auth=(self.openADS_API_login, self.openADS_API_password))
if response.status_code // 100 != 2:
error = self.get_response_error(response)
self.logger.warning("Request [GET] '%s' failed with error: '%s'" % (url, error))
self.logger.warning("Request [GET] '%s' failed with error: '%s'", url, error)
raise APIError(error)
try:
result = response.json()
@ -511,10 +510,10 @@ class AtrealOpenads(BaseResource):
}
]
url = urlparse.urljoin(self.openADS_API_url, '/dossier/%s/%s/files' % (type_dossier, numero_dossier))
response = self.requests.post(url, json=payload)
response = self.requests.post(url, json=payload, auth=(self.openADS_API_login, self.openADS_API_password))
if response.status_code // 100 != 2:
error = self.get_response_error(response)
self.logger.warning("Request [POST] '%s' failed with error: '%s'" % (url, error))
self.logger.warning("Request [POST] '%s' failed with error: '%s'", url, error)
raise APIError(error)
try:
result = response.json()
@ -583,6 +582,9 @@ class AtrealOpenads(BaseResource):
}
for fwd_file in fwd_files:
b64content = None
if fwd_file.upload_file:
b64content = base64.b64encode(fwd_file.upload_file.read())
payload.append({
'id' : fwd_file.id,
'numero_demande': fwd_file.numero_demande,
@ -593,7 +595,7 @@ class AtrealOpenads(BaseResource):
'content_type' : fwd_file.content_type,
'upload_status' : fwd_file.upload_status,
'upload_msg' : fwd_file.upload_msg,
'b64_content' : base64.b64encode(fwd_file.upload_file.read()),
'b64_content' : b64content,
'last_update_datetime' : fwd_file.last_update_datetime
})
@ -636,10 +638,10 @@ class AtrealOpenads(BaseResource):
url = urlparse.urljoin(
self.openADS_API_url,
'/dossier/%s/%s/courrier/%s' % (type_dossier, numero_dossier, 'dia_renonciation_preempter'))
response = self.requests.get(url)
response = self.requests.get(url, auth=(self.openADS_API_login, self.openADS_API_password))
if response.status_code // 100 != 2:
error = self.get_response_error(response)
self.logger.warning("Request [GET] '%s' failed with error: '%s'" % (url, error))
self.logger.warning("Request [GET] '%s' failed with error: '%s'", url, error)
raise APIError(error)
try:
result = response.json()
@ -679,7 +681,7 @@ class AtrealOpenads(BaseResource):
payload = []
fwd_files = []
for fid in file_ids:
self.logger.debug("upload_user_files() ForwardFile file_id: %s" % fid)
self.logger.debug("upload_user_files() ForwardFile file_id: %s", fid)
fwd_file = ForwardFile.objects.get(id=fid)
if fwd_file:
self.logger.debug("upload_user_files() got ForwardFile")
@ -692,29 +694,37 @@ class AtrealOpenads(BaseResource):
self.logger.debug("upload_user_files() payload added")
fwd_file.upload_status = 'uploading'
if fwd_file.upload_msg and re.search(r'^attempt \d+$', fwd_file.upload_msg):
self.logger.debug("upload_user_files() upload_msg: '%s'" % fwd_file.upload_msg)
self.logger.debug("upload_user_files() upload_msg: '%s'", fwd_file.upload_msg)
attempt_num = fwd_file.upload_msg.replace('attempt ', '').strip()
self.logger.debug("upload_user_files() attempt_num: '%s'" % attempt_num)
self.logger.debug("upload_user_files() attempt_num: '%s'", attempt_num)
fwd_file.upload_msg = 'attempt %s' % attempt_num
else:
fwd_file.upload_msg = 'attempt 1'
self.logger.debug("upload_user_files() ForwardFile ready to be saved")
fwd_file.save()
fwd_files.append(fwd_file)
else:
self.logger.warning("upload_user_files() failed to find ForwardFile file_id: %s", fid);
if payload:
self.logger.debug("upload_user_files() payload is not empty")
debug_payload = copy.deepcopy(payload)
for p in debug_payload:
if 'b64_content' in p:
p['b64_content'] = '<b64 content>'
self.logger.debug("upload_user_files() payload is: %s" % str(debug_payload))
self.logger.debug("upload_user_files() payload is: %s", str(debug_payload))
url = urlparse.urljoin(self.openADS_API_url, '/dossier/%s/%s/files' % (type_dossier, numero_dossier))
response = self.requests.post(url, json=payload)
response = self.requests.post(url, json=payload, auth=(self.openADS_API_login, self.openADS_API_password))
if response.status_code // 100 != 2:
for fwd_file in fwd_files:
fwd_file.upload_status = 'failed'
fwd_file.upload_msg = self.get_response_error(response)
fwd_file.save()
self.logger.warning(
"upload_user_files() openADS response is not OK (code: %s) for dossier '%s' and files '%s'",
response.status_code,
numero_dossier,
','.join(file_ids)
)
else:
try:
result = response.json()
@ -723,12 +733,31 @@ class AtrealOpenads(BaseResource):
fwd_file.upload_status = 'failed'
fwd_file.upload_msg = 'No JSON content returned: %r' % response.content[:1000]
fwd_file.save()
self.logger.warning(
"upload_user_files() openADS response is not JSON valid for dossier '%s' and files '%s'",
numero_dossier,
','.join([f.id for f in fwd_files])
)
else:
# TODO handle response (now its just an informational sentence in key 'data')
for fwd_file in fwd_files:
fwd_file.upload_status = 'success'
fwd_file.upload_msg = 'uploaded successfuly'
# delete file content (on success)
fpath = fwd_file.upload_file.path
fwd_file.upload_file.delete()
fwd_file.save()
self.logger.debug(
"upload_user_files() flaging file '%s' has transfered (deleted '%s')",
fwd_file.id,
fpath
)
else:
self.logger.warning(
"upload_user_files() payload is empty for dossier '%s' and files '%s'",
numero_dossier,
','.join(file_ids)
)
# copy-pasted from 'wcs/qommon/misc.py'