entrouvert
/
passerelle
15
0
Fork 0
Code Issues Pull Requests 16 Releases Activity

add logger to every requests call (#10299)

This commit is contained in:
Josue Kouka 2016-04-06 18:39:39 +02:00
parent 63786eca75
commit fd498cb687
30 changed files with 654 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
debian/debian_config.py vendored
View File

@ -25,6 +25,12 @@ LOGGING['loggers']['suds'] = {
'propagate': True,
}
LOGGING['loggers']['passerelle.resource'] = {
'level': 'DEBUG',
'handlers': ['syslog', 'mail_admins', 'sentry'],
'propagate': False,
}
# Add passerelle hobo agent
INSTALLED_APPS = ('hobo.agent.passerelle',) + INSTALLED_APPS

20
passerelle/apps/base_adresse/migrations/0003_baseaddresse_log_level.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('base_adresse', '0002_auto_20150705_0330'),
]
operations = [
migrations.AddField(
model_name='baseaddresse',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/base_adresse/migrations/0004_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('base_adresse', '0003_baseaddresse_log_level'),
]
operations = [
migrations.AlterField(
model_name='baseaddresse',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/bdp/migrations/0002_bdp_log_level.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('bdp', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='bdp',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/bdp/migrations/0003_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('bdp', '0002_bdp_log_level'),
]
operations = [
migrations.AlterField(
model_name='bdp',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

26
passerelle/apps/choosit/migrations/0002_auto_20160316_0528.py Normal file
View File

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('choosit', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='choositregistergateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
migrations.AddField(
model_name='choositsmsgateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

26
passerelle/apps/choosit/migrations/0003_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('choosit', '0002_auto_20160316_0528'),
]
operations = [
migrations.AlterField(
model_name='choositregistergateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
migrations.AlterField(
model_name='choositsmsgateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/csvdatasource/migrations/0002_csvdatasource_log_level.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('csvdatasource', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='csvdatasource',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/csvdatasource/migrations/0003_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('csvdatasource', '0002_csvdatasource_log_level'),
]
operations = [
migrations.AlterField(
model_name='csvdatasource',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/gdc/migrations/0002_gdc_log_level.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('gdc', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='gdc',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/gdc/migrations/0003_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('gdc', '0002_gdc_log_level'),
]
operations = [
migrations.AlterField(
model_name='gdc',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/mobyt/migrations/0002_mobytsmsgateway_log_level.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('mobyt', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='mobytsmsgateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/mobyt/migrations/0003_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('mobyt', '0002_mobytsmsgateway_log_level'),
]
operations = [
migrations.AlterField(
model_name='mobytsmsgateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/orange/migrations/0002_orangesmsgateway_log_level.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('orange', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='orangesmsgateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/orange/migrations/0003_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('orange', '0002_orangesmsgateway_log_level'),
]
operations = [
migrations.AlterField(
model_name='orangesmsgateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/ovh/migrations/0002_ovhsmsgateway_log_level.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('ovh', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='ovhsmsgateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/ovh/migrations/0003_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('ovh', '0002_ovhsmsgateway_log_level'),
]
operations = [
migrations.AlterField(
model_name='ovhsmsgateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/oxyd/migrations/0002_oxydsmsgateway_log_level.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('oxyd', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='oxydsmsgateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/oxyd/migrations/0003_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('oxyd', '0002_oxydsmsgateway_log_level'),
]
operations = [
migrations.AlterField(
model_name='oxydsmsgateway',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/pastell/migrations/0002_pastell_log_level.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('pastell', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='pastell',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/apps/pastell/migrations/0003_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('pastell', '0002_pastell_log_level'),
]
operations = [
migrations.AlterField(
model_name='pastell',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

30
passerelle/base/models.py
View File

@ -1,14 +1,20 @@
import logging
from django.conf import settings
from django.core.exceptions import ValidationError, ObjectDoesNotExist
from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from django.utils.text import slugify
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from model_utils.managers import InheritanceManager as ModelUtilsInheritanceManager
import passerelle
import requests
KEYTYPE_CHOICES = (
('API', 'API Key'),
('SIGN', 'Signature HMAC'),
@ -69,6 +75,19 @@ class BaseResource(models.Model):
slug = models.SlugField()
description = models.TextField()
users = models.ManyToManyField(ApiUser, blank=True)
log_level = models.CharField(
_('Log Level'), max_length=10,
choices = (
('NOTSET','NOTSET'),
('DEBUG','DEBUG'),
('INFO','INFO'),
('WARNING','WARNING'),
('ERROR','ERROR'),
('CRITICAL','CRITICAL'),
('FATAL','FATAL'),
),
default='NOTSET'
)
objects = InheritanceManager()
@ -77,6 +96,13 @@ class BaseResource(models.Model):
class Meta:
abstract = True
def __init__(self, *args, **kwargs):
super(BaseResource, self).__init__(*args, **kwargs)
self.logger = logging.getLogger('passerelle.resource.%s.%s' % (
slugify(unicode(self.__class__.__name__)), self.slug)
)
self.logger.setLevel(getattr(logging, self.log_level))
def __unicode__(self):
return self.title
@ -94,6 +120,10 @@ class BaseResource(models.Model):
return False
return getattr(settings, 'PASSERELLE_APP_%s_ENABLED' % cls._meta.app_label.upper(), False)
@property
def requests(self):
return passerelle.utils.LoggedRequest(logger=self.logger)
class AccessRight(models.Model):
codename = models.CharField(max_length=100, verbose_name='codename')

20
passerelle/datasources/migrations/0002_basedatasource_log_level.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('datasources', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='basedatasource',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/datasources/migrations/0003_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('datasources', '0002_basedatasource_log_level'),
]
operations = [
migrations.AlterField(
model_name='basedatasource',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/repost/migrations/0002_baserepost_log_level.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('repost', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='baserepost',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Debug Enabled', blank=True, choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

20
passerelle/repost/migrations/0003_auto_20160316_0910.py Normal file
View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('repost', '0002_baserepost_log_level'),
]
operations = [
migrations.AlterField(
model_name='baserepost',
name='log_level',
field=models.CharField(default=b'NOTSET', max_length=10, verbose_name='Log Level', choices=[(b'DEBUG', b'DEBUG'), (b'INFO', b'INFO')]),
preserve_default=True,
),
]

23
passerelle/settings.py
View File

@ -165,6 +165,29 @@ MELLON_IDENTITY_PROVIDERS = []
# see http://docs.python-requests.org/en/latest/user/advanced/#proxies
REQUESTS_PROXIES = {}
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
},
},
'loggers': {
'django.request': {
'handlers': ['console'],
'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
'propragate': True,
},
'passerelle.resource': {
'handlers': ['console'],
'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
'propragate': True,
},
},
}
local_settings_file = os.environ.get('PASSERELLE_SETTINGS_FILE',
os.path.join(os.path.dirname(__file__), 'local_settings.py'))
if os.path.exists(local_settings_file):

42
passerelle/utils.py
View File

@ -1,9 +1,12 @@
from functools import wraps
import json
import re
import logging
from requests import Session as RequestSession
from jsonresponse import to_json as jsonresponse_to_json
from django.conf import settings
from django.core.exceptions import PermissionDenied, ObjectDoesNotExist
from django.core.serializers.json import DjangoJSONEncoder
from django.http import HttpRequest, HttpResponse, HttpResponseBadRequest
@ -139,3 +142,42 @@ def protected_api(perm):
return view_func(instance, request, *args, **kwargs)
return _wrapped_view
return decorator
# Wrapper around requests.Session
# logging requests input and output data
class LoggedRequest(RequestSession):
def __init__(self, *args, **kwargs):
self.logger = kwargs.pop('logger')
super(LoggedRequest, self).__init__(*args, **kwargs)
def request(self, method, url, **kwargs):
self.logger.info('%s %s' % (method, url),
extra={'requests_url': url}
)
response = super(LoggedRequest, self).request(method, url, **kwargs)
self.logger.debug('Request Headers: {}'.format(''.join([
'%s: %s | ' % (k,v) for k,v in response.request.headers.items()
])))
if response.request.body:
self.logger.info('Request Payload: %r' %(response.request.body),
extra={'requests_request_payload': '%r' %response.request.body})
self.logger.info('Status code: %r' %(response.status_code),
extra={'requests_response_status': response.status_code})
resp_headers = ''.join([
'%s: %s | ' % (k,v) for k,v in response.headers.items()
])
self.logger.debug('Response Headers: %r' %resp_headers, extra={
'requests_response_headers': resp_headers})
content = response.content[:getattr(settings,
'REQUESTS_RESPONSE_CONTENT_MAX_LENGTH',5000)]
self.logger.debug('Response Content: %r' % content,
extra={'requests_response_content': content})
return response

59
tests/test_loggedrequests.py Normal file
View File

@ -0,0 +1,59 @@
import logging
import pytest
from httmock import urlmatch, HTTMock, response
from passerelle.utils import LoggedRequest
@pytest.fixture(params=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL', 'FATAL'])
def log_level(request):
return request.param
@urlmatch(netloc=r'(.*\.)?httpbin\.org$')
def httpbin_mock(url, request):
return response(200, {"message":"Are you really josh ?"}, request=request)
def test_log_level(caplog, log_level):
url = 'https://httpbin.org/post'
logger = logging.getLogger('logged_requests')
logger.setLevel(getattr(logging, log_level))
with HTTMock(httpbin_mock):
requests = LoggedRequest(logger=logger)
response = requests.post(url, json={'name':'josh'})
records = [ record for record in caplog.records() if record.name == 'logged_requests' ]
records_length = len(records)
if logger.level == 10:
assert records_length == 6
elif logger.level == 20:
assert records_length == 3
else:
assert records_length == 0
for record in records:
if record.levelname in ('DEBUG','INFO'):
if getattr(record, 'requests_url', None):
assert record.requests_url == url
if getattr(record, 'requests_request_payload', None):
assert record.requests_request_payload == '\'{"name": "josh"}\''
if getattr(record, 'requests_response_status', None):
assert record.requests_response_status == response.status_code
if record.levelname == 'DEBUG':
if getattr(record, 'requests_response_headers', None):
resp_headers = ''.join([
'%s: %s | ' % (k,v) for k,v in response.headers.items()
])
assert resp_headers == record.requests_response_headers
if getattr(record, 'requests_response_content', None):
assert record.requests_response_content == response.content

2
tox.ini
View File

@ -18,8 +18,10 @@ deps =
pytest-cov
pytest-django
pytest
pytest-capturelog
WebTest
mock
httmock
django-jsonfield >= 0.9.3
django-model-utils
pylint==1.4.0