commit 324b33edcb4cb9d156a59fc6201d396ba3a23551 Author: Benjamin Dauvergne Date: Mon Jun 30 16:09:00 2014 +0200 first commit diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..7895400 --- /dev/null +++ b/COPYING @@ -0,0 +1,2 @@ +cmsplugin-blurp is entirely under the copyright of Entr'ouvert and distributed +under the license AGPLv3 or later. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..eb762f3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include COPYING diff --git a/README b/README new file mode 100644 index 0000000..d833f63 --- /dev/null +++ b/README @@ -0,0 +1,272 @@ +Settings +======== + +You must declare blocks in a dictionnary setting named +CMS_PLUGIN_BLURP_RENDERERS, each block define a name, a renderer +class and its configuration. The key of the dictionnary define the +slug of each renderer instance, and the value associated to this +slug must be a dictionnary containing at least a key called 'name' +containing the human name of this instance. + +Renderer +======== + +A renderer is a class with the following interface:: + + class Renderer(object): + def __init__(self, slug, config): + pass + + def render(self, context, instance, placeholder): + '''Return the context to render the template''' + pass + + def render_template(self): + '''Return a template path or a Template object''' + pass + +The render method must return a context which will be passed to its +template, the render_template method must return template path or a +Django Template object. + +You can also define the following class method:: + + @classmethod + def check(cls, config) + '''Validate the config dictionnary and yield an ASCII string for each error''' + pass + +You can raise ImproperlyConfigured when the configuration does not validate. + +There is two abstract helper classes: + - `cmsplugin_blurp.renderers.base.BaseRenderer` + which provide abstract method for checking that `render()` and + `render_template()` are properly overriden and a generic + `check()` method which call the `check_config()` config method + which must return an iterator yielding strings if errors are + found + - `cmsplugin_blurp.renderers.template.TemplateRenderer` + an abstract subclass of the `BaseRenderer` which provide a + generic implementation of `render_template()` which extract the + template path from the configuration dictionnary using the key + `template_name` and if it is not found return a template parsed from the + value of the key `template`. + +Common configuration keys +========================= + +- ``ajax`` if True this key indicate to the base plugin to render the plugin using an AJAX request. + + You must add the ``cmsplugin_blurp.urls`` to your urls:: + + ... + url(r'^cmsplugin_blurp/', include('cmsplugin_blurp.urls')), + ... + +- ``ajax_refresh`` if more than zero it indicates the time between refresh of + the plugin content using the AJAX request otherwise the content is never + refreshed after the first loading. + +Static renderer +=============== + +The simplest renderer is the static renderer you can configure one like this:: + + CMS_BLURP_RENDERERS = { + 'homepage_text': { + 'name': u'Homepage text', + 'class': 'cmsplugin_blurp.renderers.static.Renderer', + 'content': u'This is the text for the homepage', + 'template': 'homepage_text.html', + } + } + +The template `homepage_text.html` could look like this:: + + {{ config.content }} + + +Data source renderer +==================== + +It load one or more local (using a `file://...` URL) or remote file (using +an `http://...` or `https://...` URL) and parses them using the +following parsers: + + - a json parser using the `json` package, + - an XML parser using the `etree.ElementTree` package, + - a RSS parser using the `feedparser` package feedparser, + - a CSV parser using the `csv` package. + +The resulting data structure can be cached, in this case loading is +asynchronous using a thread. + +The config dictonnary can contain the following keys: +- `name`, the human name of this renderer instance, +- `source`, a list of dictionnary defining the remote files, the +content of the dictionnary is described later, +- `template`, the template in which to render the data sources, it +will receive a variable named `data_sources` in its context +containing property named after the `slug` field of each source. + +A source definition is a dictionnary containing the following keys: + - `slug`, the field name to hold this source parsed value in the + template, for example with this configuration: + + + ... + 'slug': 'source1', + ... + + you can access it with this template fragment: + + {{ data_sources.source1 }} + + + - `url`, the URL of the file for this source, the scheme file://, + http://, and https:// are supported, + - `auth_mech`, whether an authentication mechanism is required by + the http[s]:// URL, it can be `hmac-sha1`, `hmac-sha256` or + `oauth2`. The HMAC mechanism is specified later; the OAuth2 + mechanisme is the classical OAuth2 HTTP bearer authentication + mechanism but it prequires that you are using django-allauth and + that an access token for the provider `authentic2` can be + retrieved for the current user, + - `signature_key`, when using the HMAC authentication mechanism it + holds the secret key used to sign the exchanges, + - `timeout`, a timeout for making the HTTP request, it is optional + and it default to 10 seconds, + - `refresh`, how long to cache the parsed value of the source, it + is optional and it defaults to 3600 seconds, + - `verify_certificate`, when the scheme of URL is https, it + indicates whether to check the SSL certificate against configured + certifate auhtorities, it is optional and defaults to True, + - `allow_redirects`, whether to follow HTTP redirects when getting + the data source file, it is optional and defaults to False, + - `parser_type`, how to parse the loaded file, it can be `json`, + `xml`, `rss`, 'csv' or 'raw' if you do not want any parsing to be + done, it is optional and defaults to 'raw', + - `content_type`, when doing an HTTP request it configures the + content of the `Accept` header, it is optional and automatically + set using the `parser_type` value. + - `limit`, when parsing an RSS file it limits the returned to first + `limit` entries sorted by date, it is optional and defaults to 0 + meaning no limit, + - `csv_params`, when parsing a csv file this dictionnary is passed + as keyword arguments to the `reader()` or `DictReader()` + constructors, depending on whether the `fieldnames` arguments is + present, + +Exemple with the JSON parser +---------------------------- + +The configuration:: + + CMS_BLURP_RENDERERS = { + 'json': { + 'name': u'My JSON content', + 'class': 'cmsplugin_blurp.renderer.data_source.Renderer', + 'sources': [ + { + 'slug': 'json_source', + 'url': 'http://example.net/file.json', + 'parser_type': 'json', + 'auth_mech': 'hmac-sha1', + 'signature_key': 'abcdefgh0123', + 'refresh': 600, + } + ] + 'template': 'my-json-block.html', + } + } + +The `my-json-block.html` template:: + +
+ {% for key, value in data_sources.json_source.iteritems %} +
{{ key }}
+
{{ value }}
+ {% endfor %} +
+ +Exemple with the CSV parser +--------------------------- + +We suppose that the file `/var/spool/data/timesheet.csv` contains +the following datas:: + + Monday,"10-12,14-17" + Tuesday,"10-12,14-18" + .... + +You can present this file using this configuration:: + + CMS_BLURP_RENDERERS = { + 'timesheet': { + 'name': u'Timesheet of our organization', + 'class': 'cmsplugin_blurp.renderer.data_source.Renderer', + 'sources': [ + { + 'slug': 'timesheet', + 'url': 'file:///var/spool/data/timesheet.csv', + 'parser_type': 'csv', + 'refresh': 86400, + 'csv_params': { + 'fieldnames': [ + 'day', + 'opening_hours', + ] + } + } + ], + 'template': 'timesheet.html', + } + } + +and the following template:: + + + + + + + {% for row in data_sources.timesheet %} + + {% endfor %} + +
DayOpening hours
{{ row.day }}{{ row.opening_hours }}
+ +SQL Renderer +============ + +Configuration:: + + CMS_BLURP_RENDERERS = { + 'student_table': { + 'name': u'Table of students', + 'class': 'cmsplugin_blurp.renderer.sql.Renderer', + 'url': 'postgresql://scott:tiger@localhost:5432/mydatabase', + 'views': { + 'students': { + 'query': 'QUERY name, age, birthdate FROM student WHERE class_id = :class_id', + 'bindparams': { + 'class_id': 12 + } + } + } + 'template': 'student-table.html', + } + } + +Template:: + + + + {% for row in students %} + + + + + + {% endfor %} +
{{ row.name }}{{ row.age }}{{ row.birthdate }}
diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..53a0304 --- /dev/null +++ b/setup.py @@ -0,0 +1,110 @@ +#! /usr/bin/env python + +''' Setup script for cmsplugin-blurp +''' + +from setuptools import setup, find_packages +from setuptools.command.install_lib import install_lib as _install_lib +from distutils.command.build import build as _build +from distutils.command.sdist import sdist as _sdist +from distutils.cmd import Command + +class compile_translations(Command): + description = 'compile message catalogs to MO files via django compilemessages' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + import os + import sys + from django.core.management.commands.compilemessages import \ + compile_messages + for path in ['src/cmsplugin_blurp/']: + curdir = os.getcwd() + os.chdir(os.path.realpath(path)) + compile_messages(sys.stderr) + os.chdir(curdir) + +class build(_build): + sub_commands = [('compile_translations', None)] + _build.sub_commands + +class sdist(_sdist): + sub_commands = [('compile_translations', None)] + _sdist.sub_commands + +class install_lib(_install_lib): + def run(self): + self.run_command('compile_translations') + _install_lib.run(self) + +def get_version(): + import glob + import re + import os + + version = None + for d in glob.glob('src/*'): + if not os.path.isdir(d): + continue + module_file = os.path.join(d, '__init__.py') + if not os.path.exists(module_file): + continue + for v in re.findall("""__version__ *= *['"](.*)['"]""", + open(module_file).read()): + assert version is None + version = v + if version: + break + assert version is not None + if os.path.exists('.git'): + import subprocess + p = subprocess.Popen(['git','describe','--dirty','--match=v*'], + stdout=subprocess.PIPE) + result = p.communicate()[0] + assert p.returncode == 0, 'git returned non-zero' + new_version = result.split()[0][1:] + assert new_version.split('-')[0] == version, '__version__ must match the last git annotated tag' + version = new_version.replace('-', '.') + return version + + +setup(name="django-cmsplugin-blurp", + version=get_version(), + license="AGPLv3 or later", + description="", + long_description=file('README').read(), + url="http://dev.entrouvert.org/projects/django-cmsplugin-blurp/", + author="Entr'ouvert", + author_email="info@entrouvert.org", + maintainer="Benjamin Dauvergne", + maintainer_email="bdauvergne@entrouvert.com", + packages=find_packages('src'), + install_requires=[], + package_dir={ + '': 'src', + }, + package_data={ + 'cmsplugin_blurp': [ + 'locale/fr/LC_MESSAGES/*.po', + 'templates/cmsplugin_blurp/*', + 'tests_data/*' + ], + }, + setup_requires=[ + 'django>=1.5', + ], + tests_require=[ + 'nose>=0.11.4', + ], + dependency_links=[], + cmdclass={ + 'build': build, + 'install_lib': install_lib, + 'compile_translations': compile_translations, + 'sdist': sdist, + }, +) diff --git a/src/cmsplugin_blurp/__init__.py b/src/cmsplugin_blurp/__init__.py new file mode 100644 index 0000000..d0476ec --- /dev/null +++ b/src/cmsplugin_blurp/__init__.py @@ -0,0 +1,2 @@ + +__version__ = '1.0.0' diff --git a/src/cmsplugin_blurp/app_settings.py b/src/cmsplugin_blurp/app_settings.py new file mode 100644 index 0000000..1d4e2ad --- /dev/null +++ b/src/cmsplugin_blurp/app_settings.py @@ -0,0 +1,17 @@ +import sys + +class AppSettings(object): + __PREFIX = 'CMS_PLUGIN_BLURP_' + __DEFAULTS = { + 'RENDERERS': (), + } + + def __getattr__(self, name): + from django.conf import settings + if name not in self.__DEFAULTS: + raise AttributeError + return getattr(settings, self.__PREFIX + name, self.__DEFAULTS[name]) + +app_settings = AppSettings() +app_settings.__name__ = __name__ +sys.modules[__name__] = app_settings diff --git a/src/cmsplugin_blurp/cms_plugins.py b/src/cmsplugin_blurp/cms_plugins.py new file mode 100644 index 0000000..6e74836 --- /dev/null +++ b/src/cmsplugin_blurp/cms_plugins.py @@ -0,0 +1,37 @@ +from django.utils.translation import ugettext_lazy as _ + +from cms.plugin_pool import plugin_pool +from cms.plugin_base import CMSPluginBase + +from . import models + +class BlurpPlugin(CMSPluginBase): + name = _('Blurp Plugin') + text_enabled = True + model = models.PluginRenderer + render_template = '' + + def render(self, context, instance, placeholder): + renderer = instance.get_renderer() + request = context.get('request') + ajax = context.get('ajaxy', True) and renderer.config.get('ajax', False) + if not ajax: + self.render_template = renderer.render_template() + return renderer.render(context, instance, placeholder) + else: + request = context.get('request') + context['plugin_id'] = instance.id + context['ajax_refresh'] = renderer.config.get('ajax_refresh', 0) + if request.GET: + context['plugin_args'] = '?{0}'.format(request.GET.urlencode()) + # hack alert !! + self.render_template = 'cmsplugin_blurp/ajax.html' + return context + + + + + + + +plugin_pool.register_plugin(BlurpPlugin) diff --git a/src/cmsplugin_blurp/locale/fr/LC_MESSAGES/django.po b/src/cmsplugin_blurp/locale/fr/LC_MESSAGES/django.po new file mode 100644 index 0000000..1e37d3c --- /dev/null +++ b/src/cmsplugin_blurp/locale/fr/LC_MESSAGES/django.po @@ -0,0 +1,37 @@ +# cmsplugin_blurp +# Copyright (C) 2014 Entr'ouvert +# This file is distributed under the same license as the cmsplugin-blurp package. +# Benjamin Dauvergne , 2014. +# +msgid "" +msgstr "" +"Project-Id-Version: cmsplugin-blurp 1.0.0\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2014-06-27 19:43+0200\n" +"PO-Revision-Date: 2014-06-30 16:06+0200\n" +"Last-Translator: Benjamin Dauvergne \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" + +#: cms_plugins.py:9 +msgid "Blurp Plugin" +msgstr "Contenu externe" + +#: models.py:11 +msgid "name" +msgstr "nom" + +#: utils.py:9 +msgid "{name} using template {template}" +msgstr "{name} via le modèle {template}" + +#: templates/cmsplugin_blurp/ajax.html:30 +msgid "loading..." +msgstr "chargement..." + +#: templates/cmsplugin_blurp/template_not_found.html:1 +msgid "Template not found" +msgstr "Template introuvable" diff --git a/src/cmsplugin_blurp/models.py b/src/cmsplugin_blurp/models.py new file mode 100644 index 0000000..59109ab --- /dev/null +++ b/src/cmsplugin_blurp/models.py @@ -0,0 +1,22 @@ +from django.db import models +from django.utils.translation import ugettext_lazy as _ + +from cms.models import CMSPlugin + +from . import utils + +class PluginRenderer(CMSPlugin): + __renderer = None + + name = models.CharField(verbose_name=_('name'), + choices=utils.renderers_choices(), + max_length=256) + + def get_renderer(self): + if self.__renderer is None: + self.__renderer = utils.resolve_renderer(self.name) + return self.__renderer + + def __unicode__(self): + return utils.renderer_description(self.get_renderer()) or self.name + diff --git a/src/cmsplugin_blurp/renderers/__init__.py b/src/cmsplugin_blurp/renderers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/cmsplugin_blurp/renderers/base.py b/src/cmsplugin_blurp/renderers/base.py new file mode 100644 index 0000000..47987d7 --- /dev/null +++ b/src/cmsplugin_blurp/renderers/base.py @@ -0,0 +1,42 @@ +import abc + +from django.core.exceptions import ImproperlyConfigured + +class BaseRenderer(object): + '''A renderer receive a configuration and provider a render + method which produce the content to render + ''' + + __metaclass__ = abc.ABCMeta + + def __init__(self, slug, config): + self.slug = slug + self.config = config + self.check(config) + + @classmethod + def check(cls, config): + errors = cls.check_config(config) + if errors is None: + return + errors = list(errors) + if errors: + raise ImproperlyConfigured('{0} configuration errors: {1} {2!r}'.format( + cls.__name__, + ', '.join(list(errors)), + config)) + + @classmethod + def check_config(cls, config): + if not 'name' in config: + yield 'name key is missing' + + @abc.abstractmethod + def render(self, context, instance, placeholder): + '''Return the context to render the template''' + pass + + @abc.abstractmethod + def render_template(self): + '''Return a template path or a Template object''' + pass diff --git a/src/cmsplugin_blurp/renderers/data_source.py b/src/cmsplugin_blurp/renderers/data_source.py new file mode 100644 index 0000000..2cd267c --- /dev/null +++ b/src/cmsplugin_blurp/renderers/data_source.py @@ -0,0 +1,307 @@ +import logging +import hashlib +from xml.etree import ElementTree as ET +import time +import threading + +import feedparser +import requests +from requests.exceptions import RequestException, HTTPError, Timeout + +from django.core.cache import cache + +from . import signature, template + +log = logging.getLogger(__name__) + + +class Renderer(template.TemplateRenderer): + '''Data source renderer the expected configuration looks like + this: + + { + 'name': u'Datas from xyz', + 'class': 'cmsplugin_blurp.renderers.data_source.Renderer', + 'sources': [ + { + 'slug': 'slug', # mandatory + 'url': 'https://...', # mandatory + 'parser_type': 'raw', # optional, possible values are json, xml, css, csv or raw, default value is raw + 'content_type': 'application/octet-stream', # optional, default value is compute from the parser_type + 'auth_mech': None, # optional, possible values are hmac-sha1, hmac-sha256,oauth2', default is None + 'signature_key': None, # mandatory if auth_mech is not None + 'verify_certificate': False, # optional, default is False + 'allow_redirects': True, # optional default is True + 'timeout': 10, # optional default is 1, it cannot be less than 1 + 'refresh': 3600, # optional, default is taken from the renderer level + 'limit': 0, # optional, default is taken from the renderer level + }, + ] + 'template_name': 'data_from_xyz.html' + # default template if the template file cannot be found + 'template': '{{ slug|pprint }}' + # time between refresh of the case + # use 0 for no cache + # cache is also update if updatecache key is in the query string + # you can also override it in each source + 'refresh': 3600, # optional default is 3600 seconds + # limit to the number of elements to return, + # not limited if it is 0 + # you can also override it in each source + 'limit': 0, # optional default is 0 + } + + ''' + + @classmethod + def check_config(cls, config): + if not 'sources' in config \ + or not isinstance(config['sources'], (tuple, list)) \ + or len(config['sources']) == 0: + yield 'sources must be a list or a tuple containing at least one element' + for source in config['sources']: + if not 'slug' in source: + yield 'each source must have a slug key' + if not 'url' in source: + yield 'each source must have an url key' + if 'parser_type' in source \ + and source['parser_type'] not in ('raw', 'csv', 'json', 'xml', 'rss'): + yield 'unknown parser_type {0!r}'.format(source['parser_type']) + if 'auth_mech' in source: + if source['auth_mech'] not in ('hmac-sha1', 'hmac-sha256', 'oauth2'): + yield 'unknown auth_mech {0!r}'.format(source['auth_mech']) + if 'signature_key' not in source or not isinstance(source['signature_key']): + yield 'missing signature_key string' + + def get_sources(self, context): + for source in self.config['sources']: + slug = '{0}.{1}'.format(self.slug, source['slug']) + data = Data(slug, self.config, source, context) + yield source['slug'], data + + def render(self, context, instance, placeholder): + for slug, source in self.get_sources(context): + context[slug] = source + return context + +class Data(object): + '''Encapsulate data from a source''' + + __CACHE_SENTINEL = object() + + JSON = 'application/json' + RSS = 'application/rss+xml' + XML = 'text/xml' + CSV = 'text/csv' + OCTET_STREAM = 'application/octet-stream' + + MAPPING = { + 'json': JSON, + 'rss': RSS, + 'xml': XML, + 'csv': CSV, + 'raw': OCTET_STREAM, + } + + + def __init__(self, slug, config, source, context): + self.slug = slug + self.context = context + self.request = context.get('request') + self.source = source + self.limit = source.get('limit', config.get('limit', 0)) + self.refresh = source.get('refresh', config.get('refresh', 0)) + self.url = source['url'] + self.verify = source.get('verify_certificate', True) + self.redirects = source.get('allow_redirects', False) + self.async = source.get('async', False) + self.timeout = source.get('timeout', 10) + self.auth_mech = source.get('auth_mech') + self.signature_key = source.get('signature_key') + self.parser_type = source.get('parser_type', 'raw') + self.content_type = source.get('content_type', self.MAPPING[self.parser_type]) + pre_hash = 'datasource-{self.slug}-{self.url}-{self.limit}-' \ + '{self.refresh}-{self.auth_mech}-{self.signature_key}' \ + .format(self=self) + self.key = hashlib.md5(pre_hash).hexdigest() + self.now = time.time() + self.__content = self.__CACHE_SENTINEL + + def get_oauth2_access_token(self): + '''Query django-allauth models to find an access token for this user''' + from allauth.socialaccount.models import SocialToken + + user = self.request.user + try: + token = SocialToken.objects.get( + account__provider='authentic2', + account__user=user) + log.debug('found access token: %r', token) + return token.token + except SocialToken.DoesNotExist: + log.warning('unable to find a social token for user: %r', user) + return '' + + def resolve_http_url(self): + try: + self.final_url = self.url + if self.source.get('auth_mech', '').startswith('hmac'): + # remove the hmac- prefix + hash_algo = self.auth_mech[:5] + self.final_url = signature.sign_url( + self.final_url, + self.signature_key, + algo=hash_algo) + log.debug('getting data source from url %r for renderer %s', + self.final_url, self.slug) + headers = { + 'Accept': self.content_type, + } + if self.auth_mech == 'oauth2': + headers['Authorization'] = 'Bearer %s' % self.get_oauth2_access_token() + request = requests.get( + self.final_url, + headers=headers, + verify=self.verify, + allow_redirects=self.redirects, + timeout=self.timeout, + stream=True) + request.raise_for_status() + return request.raw + except HTTPError: + log.warning('HTTP Error %s when loading URL %s for renderer %r', + request.status_code, + self.final_url, + self.slug) + except Timeout: + log.warning('HTTP Request timeout(%s s) when loading URL ' + '%s for renderer %s', + self.timeout, + self.final_url, + self.slug) + except RequestException: + log.warning('HTTP Request failed when loading URL ' + '%s for renderer %r', + self.final_url, + self.slug) + + def resolve_file_url(self): + path = self.url[7:] + try: + return file(path) + except Exception: + log.exception('unable to resolve file URL: %r', self.url) + + def update_content(self): + content = None + if self.url.startswith('http'): + stream = self.resolve_http_url() + elif self.url.startswith('file:'): + stream = self.resolve_file_url() + else: + log.error('unknown scheme: %r', self.url) + return + if stream is None: + return + + data = getattr(self, 'parse_'+self.parser_type)(stream) + if self.refresh and content is not None: + cache.set(self.key, (data, self.now+self.refresh), 3600) + log.debug('finished') + if self.key in self.UPDATE_THREADS: + c = self.CONDITIONS.setdefault(self.key, threading.Condition()) + with c: + self.UPDATE_THREADS.pop(self.key) + self.CONDITIONS.pop(self.key) + return data + + UPDATE_THREADS = {} + CONDITIONS = {} + + def get_content(self): + if self.__content is not self.__CACHE_SENTINEL: + return self.__content + self.__content, until = cache.get(self.key, (self.__CACHE_SENTINEL, None)) + use_cache = self.__content is not self.__CACHE_SENTINEL + # do not use cache if refresh timeout is 0 + use_cache = use_cache and self.refresh > 0 + # do not use cache if updatecache is present in the query string + use_cache = use_cache and (not self.request or 'updatecache' not in self.request.GET) + + if use_cache: + if until < self.now: + # reload cache content asynchronously in a thread + # and return the current content + log.debug('stale content reloading') + c = self.CONDITIONS.setdefault(self.key, threading.Condition()) + t = threading.Thread(target=self.update_content) + t2 = self.UPDATE_THREADS.setdefault(self.key, t) + if t2 is t: # yeah we are the first to run + with c: + t.start() + c.notify_all() # notify other updating thread that we started + if not self.async: + if not t2 is t: + with c: + while not t2.ident: + c.wait() + t2.join() + else: + self.__content = self.update_content() + return self.__content + content = property(get_content) + + def parse_json(self, stream): + import json + try: + return json.load(stream) + except ValueError, e: + log.exception('unparsable JSON content %s', e) + + def parse_rss(self, stream): + try: + result = feedparser.parse(stream.read()) + entries = result.entries + entries = sorted(result.entries, key=lambda e: e['updated_parsed']) + result.entries = entries[:self.limit] + return result + except Exception, e: + log.exception('unparsable RSS content %s', e) + + def parse_raw(self, stream): + return stream.read() + + def parse_xml(self, stream): + try: + return ET.fromstring(stream.read()) + except Exception, e: + log.exception('unparsable XML content', e) + + def parse_csv(self, stream): + import csv + + try: + params = self.source.get('csv_params', {}) + encoding = self.source.get('csv_encoding', 'utf-8') + + def list_decode(l): + return map(lambda s: s.decode(encoding), l) + + def dict_decode(d): + return dict((a, b.decode(encoding)) for a, b in d.iteritems()) + + if hasattr(stream, 'iter_lines'): + stream = stream.iter_lines() + + if 'fieldnames' in params: + reader = csv.DictReader(stream, **params) + decoder = dict_decode + else: + reader = csv.reader(stream, **params) + decoder = list_decode + return list(decoder(e) for e in reader) + except Exception, e: + log.exception('unparsable CSV content') + + def __call__(self): + return self.get_content() diff --git a/src/cmsplugin_blurp/renderers/signature.py b/src/cmsplugin_blurp/renderers/signature.py new file mode 100644 index 0000000..67e798b --- /dev/null +++ b/src/cmsplugin_blurp/renderers/signature.py @@ -0,0 +1,71 @@ +import datetime +import base64 +import hmac +import hashlib +import urllib +import random +import urlparse + +'''Simple signature scheme for query strings''' + +def sign_url(url, key, algo='sha256', timestamp=None, nonce=None): + parsed = urlparse.urlparse(url) + new_query = sign_query(parsed.query, key, algo, timestamp, nonce) + return urlparse.urlunparse(parsed[:4] + (new_query,) + parsed[5:]) + +def sign_query(query, key, algo='sha256', timestamp=None, nonce=None): + if timestamp is None: + timestamp = datetime.datetime.utcnow() + timestamp = timestamp.strftime('%Y-%m-%dT%H:%M:%SZ') + if nonce is None: + nonce = hex(random.getrandbits(128))[2:] + new_query = query + if new_query: + new_query += '&' + new_query += urllib.urlencode(( + ('algo', algo), + ('timestamp', timestamp), + ('nonce', nonce))) + signature = base64.b64encode(sign_string(new_query, key, algo=algo)) + new_query += '&signature=' + urllib.quote(signature) + return new_query + +def sign_string(s, key, algo='sha256', timedelta=30): + digestmod = getattr(hashlib, algo) + hash = hmac.HMAC(key, digestmod=digestmod, msg=s) + return hash.digest() + +def check_url(url, key, known_nonce=None, timedelta=30): + parsed = urlparse.urlparse(url, 'https') + return check_query(parsed.query, key) + +def check_query(query, key, known_nonce=None, timedelta=30): + parsed = urlparse.parse_qs(query) + signature = base64.b64decode(parsed['signature'][0]) + algo = parsed['algo'][0] + timestamp = parsed['timestamp'][0] + timestamp = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ') + nonce = parsed['nonce'] + unsigned_query = query.split('&signature=')[0] + if known_nonce is not None and known_nonce(nonce): + return False + print 'timedelta', datetime.datetime.utcnow() - timestamp + if abs(datetime.datetime.utcnow() - timestamp) > datetime.timedelta(seconds=timedelta): + return False + return check_string(unsigned_query, signature, key, algo=algo) + +def check_string(s, signature, key, algo='sha256'): + # constant time compare + signature2 = sign_string(s, key, algo=algo) + if len(signature2) != len(signature): + return False + res = 0 + for a, b in zip(signature, signature2): + res |= ord(a) ^ ord(b) + return res == 0 + +if __name__ == '__main__': + test_key = '12345' + signed_query = sign_query('NameId=_12345&orig=montpellier', test_key) + assert check_query(signed_query, test_key, timedelta=0) is False + assert check_query(signed_query, test_key) is True diff --git a/src/cmsplugin_blurp/renderers/sql.py b/src/cmsplugin_blurp/renderers/sql.py new file mode 100644 index 0000000..becb663 --- /dev/null +++ b/src/cmsplugin_blurp/renderers/sql.py @@ -0,0 +1,31 @@ +import logging + +from sqlalchemy import create_engine +from sqlalchemy.sql import text, bindparam +from sqlalchemy.pool import NullPool + +from . import template + +log = logging.getLogger(__name__) + +class Renderer(template.Renderer): + def __init__(self, *args, **kwargs): + super(Renderer, self).__init__(*args, **kwargs) + self.engine = create_engine(self.config['url'], + poolclass=NullPool, **self.config.get('kwargs', {})) + self.views = self.config['views'] + + def render(self, context, instance, placeholder): + for view in self.views: + query = view['query'] + bindparams = [] + for name, value in view.get('bindparams', {}).iteritems(): + param = bindparam(name, value=value) + bindparams.append(param) + sql = text(query, bindparams=bindparams) + result = self.engine.execute(sql) + keys = result.keys() + result = [dict(zip(keys, row)) for row in result.fetchall()] + context[view['slug']] = result + return context + diff --git a/src/cmsplugin_blurp/renderers/static.py b/src/cmsplugin_blurp/renderers/static.py new file mode 100644 index 0000000..f9239bb --- /dev/null +++ b/src/cmsplugin_blurp/renderers/static.py @@ -0,0 +1,8 @@ +from .template import TemplateRenderer + +class Renderer(TemplateRenderer): + '''Directly pass the config object to the template''' + + def render(self, context, instance, placeholder): + context.update(self.config) + return context diff --git a/src/cmsplugin_blurp/renderers/template.py b/src/cmsplugin_blurp/renderers/template.py new file mode 100644 index 0000000..772551d --- /dev/null +++ b/src/cmsplugin_blurp/renderers/template.py @@ -0,0 +1,35 @@ +import logging + +from django.core.exceptions import ImproperlyConfigured +from django.template.loader import get_template +from django.template import TemplateDoesNotExist, Template + +from .base import BaseRenderer + +log = logging.getLogger(__name__) + +class TemplateRenderer(BaseRenderer): + '''Base class providing basic functionalities to resolve the template from + the renderer configuration. + ''' + + @classmethod + def check_config(cls, config): + super(TemplateRenderer, cls).check_config(config) + if not 'template' in config and not 'template_name' in config: + raise ImproperlyConfigured('{0} configuration is missing a template key: {1!r}'.format( + cls.__name__, config)) + + def render_template(self): + '''First try to get a template by path, then compile the inline + template, and if none of that works show an error message.''' + + if 'template_name' in self.config: + try: + return get_template(self.config['template_name']) + except TemplateDoesNotExist: + pass + if 'template' in self.config: + return Template(self.config['template']) + log.error('template not found: %r', self.config) + return 'cmsplugin_blurp/template_not_found.html' diff --git a/src/cmsplugin_blurp/templates/cmsplugin_blurp/ajax.html b/src/cmsplugin_blurp/templates/cmsplugin_blurp/ajax.html new file mode 100644 index 0000000..7abe443 --- /dev/null +++ b/src/cmsplugin_blurp/templates/cmsplugin_blurp/ajax.html @@ -0,0 +1,40 @@ +{% load i18n %} +{% load sekizai_tags %} + +{% addtoblock "js" %} + +{% endaddtoblock %} + +
+
{% trans "loading..." %}
+
+ + diff --git a/src/cmsplugin_blurp/templates/cmsplugin_blurp/sekizai_render.html b/src/cmsplugin_blurp/templates/cmsplugin_blurp/sekizai_render.html new file mode 100644 index 0000000..7fa9ee7 --- /dev/null +++ b/src/cmsplugin_blurp/templates/cmsplugin_blurp/sekizai_render.html @@ -0,0 +1,4 @@ +{% load sekizai_tags %} +{{ content|safe }} +{% render_block "js" %} +{% render_block "css" %} diff --git a/src/cmsplugin_blurp/templates/cmsplugin_blurp/template_not_found.html b/src/cmsplugin_blurp/templates/cmsplugin_blurp/template_not_found.html new file mode 100644 index 0000000..fd653ca --- /dev/null +++ b/src/cmsplugin_blurp/templates/cmsplugin_blurp/template_not_found.html @@ -0,0 +1 @@ +{% load i18n %}{% trans "Template not found" %} diff --git a/src/cmsplugin_blurp/tests.py b/src/cmsplugin_blurp/tests.py new file mode 100644 index 0000000..3d09847 --- /dev/null +++ b/src/cmsplugin_blurp/tests.py @@ -0,0 +1,80 @@ +import os.path + +from django.test import TestCase +from django.test.utils import override_settings + +from django.template import Context + +from cmsplugin_blurp import utils + +BASE_FILE = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + 'tests_data')) + +CMS_PLUGIN_BLURP_RENDERERS = { + 'static': { + 'template': 'test.html', + 'class': 'cmsplugin_blurp.renderers.static.Renderer', + 'content': 'xxx', + }, +} +for kind in ('raw', 'json', 'rss', 'xml'): + CMS_PLUGIN_BLURP_RENDERERS[kind] = { + 'template': 'test.html', + 'class': 'cmsplugin_blurp.renderers.data_source.Renderer', + 'sources': [ + { + 'slug': kind, + 'parser_type': kind, + 'url': 'file://' + os.path.join(BASE_FILE, kind), + } + ] + } + +@override_settings(CMS_PLUGIN_BLURP_RENDERERS=CMS_PLUGIN_BLURP_RENDERERS) +class RendererTestCase(TestCase): + def test_choices(self): + self.assertEqual(set(utils.renderers_choices()), + set([('static', 'static'), + ('raw', 'raw'), + ('json', 'json'), + ('rss', 'rss'), + ('xml', 'xml'), + ])) + + def test_static_renderer(self): + r = utils.resolve_renderer('static') + self.assertIsNotNone(r) + c = r.render(Context(), None, None) + self.assertTrue(c.has_key('content')) + self.assertTrue(c.has_key('template')) + self.assertEqual(c['content'], 'xxx') + + def test_data_source_renderer_raw(self): + r = utils.resolve_renderer('raw') + self.assertIsNotNone(r) + c = r.render(Context(), None, None) + self.assertTrue(c.has_key('raw')) + self.assertEqual(unicode(c['raw']()), 'xxx') + + def test_data_source_renderer_json(self): + r = utils.resolve_renderer('json') + self.assertIsNotNone(r) + c = r.render(Context(), None, None) + self.assertTrue(c.has_key('json')) + self.assertEqual(c['json'](), {'xxx':'yyy'}) + + def test_data_source_renderer_rss(self): + r = utils.resolve_renderer('rss') + self.assertIsNotNone(r) + c = r.render(Context(), None, None) + self.assertTrue(c.has_key('rss')) + self.assertIn('feed', c['rss']()) + + def test_data_source_renderer_xml(self): + r = utils.resolve_renderer('xml') + self.assertIsNotNone(r) + c = r.render(Context(), None, None) + self.assertTrue(c.has_key('xml')) + self.assertEqual(c['xml']().tag, 'html') diff --git a/src/cmsplugin_blurp/tests_data/json b/src/cmsplugin_blurp/tests_data/json new file mode 100644 index 0000000..b074e4c --- /dev/null +++ b/src/cmsplugin_blurp/tests_data/json @@ -0,0 +1,3 @@ +{ + "xxx": "yyy" +} diff --git a/src/cmsplugin_blurp/tests_data/raw b/src/cmsplugin_blurp/tests_data/raw new file mode 100644 index 0000000..ac8522f --- /dev/null +++ b/src/cmsplugin_blurp/tests_data/raw @@ -0,0 +1 @@ +xxx \ No newline at end of file diff --git a/src/cmsplugin_blurp/tests_data/rss b/src/cmsplugin_blurp/tests_data/rss new file mode 100644 index 0000000..6843759 --- /dev/null +++ b/src/cmsplugin_blurp/tests_data/rss @@ -0,0 +1,95 @@ + + + + <![CDATA[Människor och minnen]]> + http://areena.yle.fi/tv/1658066 + Mon, 16 Jun 2014 10:05:00 +0300 + + + + http://areena.yle.fi/static/mk/images/areena/series/3318611_886_manniskor_och_minnen_65.jpg + <![CDATA[Människor och minnen]]> + http://areena.yle.fi/tv/1658066 + + (areena.info@yle.fi) + areena.info@yle.fi + YLE Areena + YLE Areena + fi + http://blogs.law.harvard.edu/tech/rss + 15 + areena.info@yle.fi + + areena.info@yle.fi + + + + <![CDATA[Människor och minnen: Tre män och en velociped]]> + Yle Radio Vega + false + http://areena.yle.fi/tv/2231861 + 09e60bbfad60423cb2cc1906153cba0c + + Mon, 16 Jun 2014 11:00:00 +0300 + Fakta ja kulttuuri + + + + + + + + + start=2014-06-16T11:00:00+0300; end=2014-07-16T23:59:59+0300; scheme=W3C-DTF; + + + + <![CDATA[Människor och minnen: De 99 minareternas stad]]> + Yle Radio Vega + false + http://areena.yle.fi/tv/2270731 + 68115c4dfece4893b6c7fd43b45233e6 + + Mon, 9 Jun 2014 11:00:00 +0300 + Fakta ja kulttuuri + + + + + + + + + start=2014-06-09T11:00:00+0300; end=2014-07-09T23:59:59+0300; scheme=W3C-DTF; + + + + <![CDATA[Människor och minnen: Författaren och båten]]> + Yle Radio Vega + false + http://areena.yle.fi/tv/2266940 + 4e5da61ca5904810b530ef72551ece02 + + Mon, 2 Jun 2014 11:00:00 +0300 + Fakta ja kulttuuri + + + + + + + + + + start=2014-06-02T11:00:00+0300; end=2014-07-02T23:59:59+0300; scheme=W3C-DTF; + + + + + diff --git a/src/cmsplugin_blurp/tests_data/xml b/src/cmsplugin_blurp/tests_data/xml new file mode 100644 index 0000000..18ecdcb --- /dev/null +++ b/src/cmsplugin_blurp/tests_data/xml @@ -0,0 +1 @@ + diff --git a/src/cmsplugin_blurp/urls.py b/src/cmsplugin_blurp/urls.py new file mode 100644 index 0000000..4cb18f3 --- /dev/null +++ b/src/cmsplugin_blurp/urls.py @@ -0,0 +1,9 @@ +from django.conf.urls import patterns, url + +from . import views + +urlpatterns = patterns('', + url(r'^block-plugin-async/(?P\d+)/$', + views.ajax_render, + name='ajax_render') + ) diff --git a/src/cmsplugin_blurp/utils.py b/src/cmsplugin_blurp/utils.py new file mode 100644 index 0000000..a4fc5cd --- /dev/null +++ b/src/cmsplugin_blurp/utils.py @@ -0,0 +1,26 @@ +from django.utils.importlib import import_module +from django.utils.translation import ugettext_lazy as _ + +from . import app_settings + +def renderer_description(renderer): + if 'name' in renderer.config: + if 'template_name' in renderer.config: + return _('{name} using template {template}').format( + name=renderer.config['name'], + template=renderer.config['template_name']) + else: + return renderer.config['name'] + +def renderers_choices(): + for slug in app_settings.RENDERERS: + renderer = resolve_renderer(slug) + yield slug, renderer_description(renderer) or slug + +def resolve_renderer(name): + '''Create a renderer instance from slug name of its settings''' + instance = app_settings.RENDERERS.get(name) + if instance: + module_name, class_name = instance['class'].rsplit('.', 1) + module = import_module(module_name) + return getattr(module, class_name)(name, instance) diff --git a/src/cmsplugin_blurp/views.py b/src/cmsplugin_blurp/views.py new file mode 100644 index 0000000..30a5057 --- /dev/null +++ b/src/cmsplugin_blurp/views.py @@ -0,0 +1,18 @@ +import json + +from django.http import HttpResponse +from django.template import RequestContext, loader + +from cms.models import CMSPlugin + +def ajax_render(request, plugin_id): + plugin = CMSPlugin.objects.get(pk=plugin_id) + context = RequestContext(request) + context['ajaxy'] = False + rendered = plugin.render_plugin(context) + # use another template to render accumulated js and css declarations from sekizai + content = loader.render_to_string('cmsplugin_blurp/sekizai_render.html', + {'content': rendered}, + context) + return HttpResponse(json.dumps({'content': content})) +