From 42bd357c9697c8cb22b4504a631261a055deadac Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Mon, 27 Jan 2014 11:21:33 +0100 Subject: [PATCH] data_source_plugin: add DataSource.timeout field to limit the time taken to download a data source --- .../apps/data_source_plugin/cms_plugins.py | 3 +- ...0005_auto__add_field_datasource_timeout.py | 77 +++++++++++++++++++ .../apps/data_source_plugin/models.py | 4 + 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 portail_citoyen/apps/data_source_plugin/migrations/0005_auto__add_field_datasource_timeout.py diff --git a/portail_citoyen/apps/data_source_plugin/cms_plugins.py b/portail_citoyen/apps/data_source_plugin/cms_plugins.py index ef91043..9594d83 100644 --- a/portail_citoyen/apps/data_source_plugin/cms_plugins.py +++ b/portail_citoyen/apps/data_source_plugin/cms_plugins.py @@ -58,7 +58,8 @@ class Data(object): } request = requests.get(self.final_url, headers=headers, verify=self.data_source.verify_certificate, - allow_redirects=self.data_source.allow_redirects) + allow_redirects=self.data_source.allow_redirects, + timeout=self.data_source.timeout) request.raise_for_status() except HTTPError: logger.warning('HTTP Error %s when loading datasource %s from' diff --git a/portail_citoyen/apps/data_source_plugin/migrations/0005_auto__add_field_datasource_timeout.py b/portail_citoyen/apps/data_source_plugin/migrations/0005_auto__add_field_datasource_timeout.py new file mode 100644 index 0000000..41433a5 --- /dev/null +++ b/portail_citoyen/apps/data_source_plugin/migrations/0005_auto__add_field_datasource_timeout.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'DataSource.timeout' + db.add_column(u'data_source_plugin_datasource', 'timeout', + self.gf('django.db.models.fields.IntegerField')(default=10), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'DataSource.timeout' + db.delete_column(u'data_source_plugin_datasource', 'timeout') + + + models = { + 'cms.cmsplugin': { + 'Meta': {'object_name': 'CMSPlugin'}, + 'changed_date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'creation_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'language': ('django.db.models.fields.CharField', [], {'max_length': '15', 'db_index': 'True'}), + 'level': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}), + 'lft': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}), + 'parent': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cms.CMSPlugin']", 'null': 'True', 'blank': 'True'}), + 'placeholder': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cms.Placeholder']", 'null': 'True'}), + 'plugin_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}), + 'position': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'rght': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}), + 'tree_id': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}) + }, + 'cms.placeholder': { + 'Meta': {'object_name': 'Placeholder'}, + 'default_width': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'slot': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}) + }, + u'data_source_plugin.datasource': { + 'Meta': {'ordering': "('name',)", 'object_name': 'DataSource'}, + 'allow_redirects': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_algo': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '16', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'signature_key': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128', 'blank': 'True'}), + 'timeout': ('django.db.models.fields.IntegerField', [], {'default': '10'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '1024'}), + 'verify_certificate': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + u'data_source_plugin.datasourceplugin': { + 'Meta': {'object_name': 'DataSourcePlugin', 'db_table': "u'cmsplugin_datasourceplugin'"}, + u'cmsplugin_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['cms.CMSPlugin']", 'unique': 'True', 'primary_key': 'True'}), + 'limit': ('django.db.models.fields.IntegerField', [], {'default': '10'}), + 'refresh': ('django.db.models.fields.IntegerField', [], {'default': '60'}), + 'template_source': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}) + }, + u'data_source_plugin.plugindatasource': { + 'Meta': {'ordering': "('order', 'id')", 'object_name': 'PluginDataSource'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'order': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'plugin': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'sources'", 'to': u"orm['data_source_plugin.DataSourcePlugin']"}), + 'source': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'plugins'", 'to': u"orm['data_source_plugin.DataSource']"}) + }, + u'data_source_plugin.rawinlinetemplateplugin': { + 'Meta': {'object_name': 'RawInlineTemplatePlugin', 'db_table': "u'cmsplugin_rawinlinetemplateplugin'"}, + u'cmsplugin_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['cms.CMSPlugin']", 'unique': 'True', 'primary_key': 'True'}), + 'template_source': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}) + } + } + + complete_apps = ['data_source_plugin'] \ No newline at end of file diff --git a/portail_citoyen/apps/data_source_plugin/models.py b/portail_citoyen/apps/data_source_plugin/models.py index d0042c2..953686f 100644 --- a/portail_citoyen/apps/data_source_plugin/models.py +++ b/portail_citoyen/apps/data_source_plugin/models.py @@ -48,6 +48,10 @@ class DataSource(models.Model): allow_redirects = models.BooleanField(verbose_name=_('allows HTTP redirections'), help_text=_('it can improve latencies to forbid redirection follow'), default=True) + timeout = models.IntegerField(verbose_name=_('timeout'), + default=10, + help_text=_('time in second to wait before ' + 'failing to download a datasource')) def clean(self): if self.signature_key and not self.hash_algo: