wcs/wcs/api.py

# w.c.s. - web application for online forms
# Copyright (C) 2005-2013  Entr'ouvert
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

import base64
import hmac
import hashlib
import json
import datetime
import time
import urllib
import urllib2
import urlparse
import random
import sys

from quixote import get_request, get_publisher, get_response
from quixote.directory import Directory
from qommon import misc
from qommon.errors import (AccessForbiddenError, QueryError, TraversalError,
    UnknownNameIdAccessForbiddenError)

from wcs.categories import Category
from wcs.formdef import FormDef
from wcs.roles import Role, logged_users_role
from wcs.forms.common import FormStatusPage
from wcs.forms.root import RootDirectory
import wcs.qommon.storage as st


def is_url_signed():
    query_string = get_request().get_query()
    if not query_string:
        return False
    signature = get_request().form.get('signature')
    if not isinstance(signature, basestring):
        return False
    # verify signature
    orig = get_request().form.get('orig')
    if not isinstance(orig, basestring):
        raise AccessForbiddenError('missing/multiple orig field')
    key = get_publisher().get_site_option(orig, 'api-secrets')
    if not key:
        raise AccessForbiddenError('invalid orig')
    algo = get_request().form.get('algo')
    if not isinstance(algo, basestring):
        raise AccessForbiddenError('missing/multiple algo field')
    try:
        algo = getattr(hashlib, algo)
    except AttributeError:
        raise AccessForbiddenError('invalid algo')
    if signature != base64.standard_b64encode(hmac.new(key,
                            query_string[:query_string.find('&signature=')],
                            algo).digest()):
        raise AccessForbiddenError('invalid signature')
    timestamp = get_request().form.get('timestamp')
    if not isinstance(timestamp, basestring):
        raise AccessForbiddenError('missing/multiple timestamp field')
    delta = (datetime.datetime.utcnow().replace(tzinfo=None) -
             datetime.datetime.strptime(timestamp,
                     '%Y-%m-%dT%H:%M:%SZ'))
    MAX_DELTA = 30
    if abs(delta) > datetime.timedelta(seconds=MAX_DELTA):
        raise AccessForbiddenError('timestamp delta is more '
                'than %s seconds: %s seconds' % (MAX_DELTA, delta))
    return True

def get_user_from_api_query_string():
    if not is_url_signed():
        return None
    # Signature is good. Now looking for the user, by email/NameID.
    # If email or NameID exist but are empty, return None
    user = None
    if get_request().form.get('email'):
        email = get_request().form.get('email')
        if not isinstance(email, basestring):
            raise AccessForbiddenError('multiple email field')
        users = list(get_publisher().user_class.get_users_with_email(email))
        if users:
            user = users[0]
        else:
            raise AccessForbiddenError('unknown email')
    elif get_request().form.get('NameID'):
        ni = get_request().form.get('NameID')
        if not isinstance(ni, basestring):
            raise AccessForbiddenError('multiple NameID field')
        users = list(get_publisher().user_class.get_users_with_name_identifier(ni))
        if users:
            user = users[0]
        else:
            raise UnknownNameIdAccessForbiddenError('unknown NameID')

    return user

def sign_url(url, key, algo='sha256', timestamp=None, nonce=None):
    parsed = urlparse.urlparse(url)
    new_query = sign_query(parsed.query, key, algo, timestamp, nonce)
    return urlparse.urlunparse(parsed[:4] + (new_query,) + parsed[5:])

def sign_query(query, key, algo='sha256', timestamp=None, nonce=None):
    if timestamp is None:
        timestamp = datetime.datetime.utcnow()
    timestamp = timestamp.strftime('%Y-%m-%dT%H:%M:%SZ')
    if nonce is None:
        nonce = hex(random.getrandbits(128))[2:-1]
    new_query = query
    if new_query:
        new_query += '&'
    new_query += urllib.urlencode((
        ('algo', algo),
        ('timestamp', timestamp),
        ('nonce', nonce)))
    signature = base64.b64encode(sign_string(new_query, key, algo=algo))
    new_query += '&signature=' + urllib.quote(signature)
    return new_query

def sign_string(s, key, algo='sha256', timedelta=30):
    digestmod = getattr(hashlib, algo)
    hash = hmac.HMAC(key, digestmod=digestmod, msg=s)
    return hash.digest()


# import backoffice.root.FormPage after get_user_from_api_query_string
# to avoid circular dependencies
from backoffice.management import FormPage as BackofficeFormPage


class ApiFormdataPage(FormStatusPage):
    _q_exports = ['', 'download']

    def _q_index(self):
        return self.json()

    def check_receiver(self):
        api_user = get_user_from_api_query_string()
        if not api_user:
            if get_request().user and get_request().user.is_admin:
                return # grant access to admins, to ease debug
            raise AccessForbiddenError()
        if not self.formdef.is_user_allowed_read_status_and_history(api_user, self.filled):
            raise AccessForbiddenError()


class ApiFormPage(BackofficeFormPage):
    _q_exports = [('list', 'json')] # same as backoffice but restricted to json export

    def __init__(self, component):
        try:
            self.formdef = FormDef.get_by_urlname(component)
        except KeyError:
            raise TraversalError()

        api_user = get_user_from_api_query_string()
        if not api_user:
            if get_request().user and get_request().user.is_admin:
                return # grant access to admins, to ease debug
            raise AccessForbiddenError()
        if not self.formdef.is_of_concern_for_user(api_user):
            raise AccessForbiddenError()

    def _q_lookup(self, component):
        try:
            formdata = self.formdef.data_class().get(component)
        except KeyError:
            raise TraversalError()
        return ApiFormdataPage(self.formdef, formdata)


class ApiFormsDirectory(Directory):
    def _q_lookup(self, component):
        api_user = get_user_from_api_query_string()
        if not api_user:
            # grant access to admins, to ease debug
            if not (get_request().user and get_request().user.is_admin):
                raise AccessForbiddenError()
        return ApiFormPage(component)


class ApiFormdefDirectory(Directory):
    _q_exports = ['schema', 'submit']

    def __init__(self, formdef):
        self.formdef = formdef

    def schema(self):
        get_response().set_content_type('application/json')
        return self.formdef.export_to_json()

    def submit(self):
        # expects json as input
        #  {
        #   "meta": {
        #      "attr": "value"
        #   },
        #   "data": {
        #      "0": "value",
        #      "1": "value",
        #      ...
        #   }
        #  }
        get_response().set_content_type('application/json')
        user = get_user_from_api_query_string()
        if not user:
            raise AccessForbiddenError('no user')
        if self.formdef.is_disabled():
            raise AccessForbiddenError('disabled form')
        json_input = get_request().json
        formdata = self.formdef.data_class()()
        formdata.data = json_input['data']
        meta = json_input.get('meta') or {}
        if meta.get('backoffice-submission'):
            if not self.formdef.backoffice_submission_roles:
                raise AccessForbiddenError('no backoffice submission roles')
            if not set(user.roles or []).intersection(self.formdef.backoffice_submission_roles):
                raise AccessForbiddenError('not cleared for backoffice submit')
            formdata.backoffice_submission = True
        else:
            formdata.user_id = user.id
        if json_input.get('context'):
            formdata.submission_context = json_input['context']
        formdata.store()
        if self.formdef.enable_tracking_codes:
            code = get_publisher().tracking_code_class()
            code.formdata = formdata # this will .store() the code
        if meta.get('draft'):
            formdata.status = 'draft'
            formdata.receipt_time = time.localtime()
            formdata.store()
        else:
            formdata.just_created()
            formdata.store()
            formdata.perform_workflow()
            formdata.store()
        return json.dumps({'err': 0, 'data': {'id': formdata.id}})


class ApiFormdefsDirectory(Directory):
    _q_exports = ['']

    def __init__(self, category=None):
        self.category = category

    def _q_index(self):
        try:
            user = get_user_from_api_query_string() or get_request().user
        except UnknownNameIdAccessForbiddenError:
            # if authenticating the user via the query string failed, return
            # results for the anonymous case; user is set to 'False' as a
            # signed URL with a None user is considered like an appropriate
            # webservice call.
            user = False
        list_all_forms = (user and user.is_admin) or (is_url_signed() and user is None)

        list_forms = []

        if self.category:
            formdefs = FormDef.select(lambda x: (
                        str(x.category_id) == str(self.category.id) and (
                            not x.is_disabled() or x.disabled_redirection)),
                        order_by = 'name')
        else:
            formdefs = FormDef.select(lambda x: not x.is_disabled() or x.disabled_redirection,
                        order_by='name',
                        ignore_errors=True)

        charset = get_publisher().site_charset

        for formdef in formdefs:
            authentication_required = False
            if formdef.roles and not list_all_forms:
                if not user:
                    if not formdef.always_advertise:
                        continue
                    authentication_required = True
                elif logged_users_role().id not in formdef.roles:
                    for q in user.roles or []:
                        if q in formdef.roles:
                            break
                    else:
                        if not formdef.always_advertise:
                            continue
                        authentication_required = True

            if formdef.keywords:
                keywords = [x.strip() for x in formdef.keywords.split(',')]
            else:
                keywords = []

            formdict = {'title': unicode(formdef.name, charset),
                        'slug': formdef.url_name,
                        'url': formdef.get_url(),
                        'description': formdef.description or '',
                        'keywords': keywords,
                        'authentication_required': authentication_required}

            formdict['redirection'] = bool(formdef.is_disabled() and
                    formdef.disabled_redirection)

            # we include the count of submitted forms so it's possible to sort
            # them by popularity
            formdict['count'] = formdef.data_class().count()

            formdict['functions'] = {}
            formdef_workflow_roles = formdef.workflow_roles or {}
            for (wf_role_id, wf_role_label) in formdef.workflow.roles.items():
                workflow_function = {'label': wf_role_label}
                role_id = formdef_workflow_roles.get(wf_role_id)
                if role_id:
                    try:
                        workflow_function['role'] = Role.get(role_id).get_json_export_dict()
                    except KeyError:
                        pass
                formdict['functions'][wf_role_id] = workflow_function

            if formdef.category:
                formdict['category'] = unicode(formdef.category.name, charset)
                formdict['category_position'] = (formdef.category.position or 0)
            else:
                formdict['category_position'] = sys.maxint

            list_forms.append(formdict)

        list_forms.sort(lambda x, y: cmp(x['category_position'], y['category_position']))
        for formdict in list_forms:
            del formdict['category_position']

        get_response().set_content_type('application/json')
        return json.dumps(list_forms)

    def _q_lookup(self, component):
        return ApiFormdefDirectory(FormDef.get_by_urlname(component))


class ApiCategoryDirectory(Directory):
    _q_exports = ['formdefs']

    def __init__(self, category):
        self.category = category
        self.formdefs = ApiFormdefsDirectory(category)


class ApiCategoriesDirectory(RootDirectory):
    _q_exports = ['']

    def __init__(self):
        pass

    def _q_index(self):
        try:
            user = get_user_from_api_query_string() or get_request().user
        except UnknownNameIdAccessForbiddenError:
            # the name id was unknown, return the categories for anonymous
            # users.
            user = None
        list_categories = []
        charset = get_publisher().site_charset
        categories = self.get_categories(user)
        Category.sort_by_position(categories)
        for category in categories:
            d = {}
            d['title'] = unicode(category.name, charset)
            d['slug'] = category.url_name
            d['url'] = category.get_url()
            if category.description:
                d['description'] = unicode(category.description, charset)
            list_categories.append(d)
        get_response().set_content_type('application/json')
        return json.dumps({'data': list_categories})

    def _q_lookup(self, component):
        return ApiCategoryDirectory(Category.get_by_urlname(component))


class ApiUserDirectory(Directory):
    _q_exports = ['', 'forms', 'drafts']

    def __init__(self, user=None):
        self.user = user

    def _q_index(self):
        get_response().set_content_type('application/json')
        user = self.user or get_user_from_api_query_string() or get_request().user
        if not user:
            raise AccessForbiddenError('no user specified')
        user_info = user.get_substitution_variables(prefix='')
        del user_info['user']
        user_info['user_roles'] = [
                Role.get(x).get_json_export_dict() for x in user.roles or []]
        return json.dumps(user_info)

    def get_user_forms(self, user):
        formdefs = FormDef.select(lambda x: not x.is_disabled())
        user_forms = []
        for formdef in formdefs:
            user_forms.extend(formdef.data_class().get_with_indexed_value(
                        'user_id', user.id))
            try:
                user_forms.extend(formdef.data_class().get_with_indexed_value(
                        'user_hash', user.hash))
            except AttributeError:
                pass
        user_forms.sort(lambda x, y: cmp(x.receipt_time, y.receipt_time))
        return user_forms

    def drafts(self):
        get_response().set_content_type('application/json')
        user = self.user or get_user_from_api_query_string() or get_request().user
        if not user:
            raise AccessForbiddenError()
        drafts = []
        for form in self.get_user_forms(user):
            if not form.is_draft():
                continue
            title = _('%(name)s, draft saved on %(datetime)s') % {
                'name': form.formdef.name,
                'datetime': misc.localstrftime(form.receipt_time)
            }
            # !!! no trailing slash in the special draft case
            url = form.get_url().rstrip('/')
            d = {'title': title,
                'name': form.formdef.name,
                'url': url,
                'datetime': misc.strftime.strftime('%Y-%m-%d %H:%M:%S', form.receipt_time),
            }
            if get_request().form.get('full') == 'on':
                d.update(form.get_json_export_dict(include_files=False))
            drafts.append(d)

        return json.dumps(drafts,
                cls=misc.JSONEncoder,
                encoding=get_publisher().site_charset)

    def forms(self):
        get_response().set_content_type('application/json')
        user = self.user or get_user_from_api_query_string() or get_request().user
        if not user:
            raise AccessForbiddenError()
        forms = []
        for form in self.get_user_forms(user):
            if form.is_draft():
                continue
            visible_status = form.get_visible_status(user=user)
            # skip hidden forms
            if not visible_status:
                continue
            name = form.formdef.name
            id = form.get_display_id()
            status = visible_status.name
            title = _('%(name)s #%(id)s (%(status)s)') % {
                    'name': name,
                    'id': id,
                    'status': status
            }
            url = form.get_url()
            d = {'title': title,
                'name': form.formdef.name,
                'url': url,
                'datetime': misc.strftime.strftime('%Y-%m-%d %H:%M:%S', form.receipt_time),
                'status': status,
            }
            d.update(form.get_substitution_variables(minimal=True))
            if get_request().form.get('full') == 'on':
                d.update(form.get_json_export_dict(include_files=False))
            forms.append(d)

        return json.dumps(forms,
                cls=misc.JSONEncoder,
                encoding=get_publisher().site_charset)


class ApiUsersDirectory(Directory):
    _q_exports = ['']

    def _q_index(self):
        get_response().set_content_type('application/json')
        if not (is_url_signed() or (
                get_request().user and get_request().user.can_go_in_admin())):
            raise AccessForbiddenError()

        criterias = []
        query = get_request().form.get('q')
        if query:
            criterias.append(st.Or([st.ILike('name', query), st.ILike('email', query)]))

        def as_dict(user):
            user_info = user.get_substitution_variables(prefix='')
            del user_info['user']
            user_info['user_id'] = user.id
            return user_info

        limit = int(get_request().form.get('limit') or '0') or None
        users = get_publisher().user_class.select(order_by='name',
                clause=criterias, limit=limit)
        data = [as_dict(x) for x in users]
        return json.dumps({'data': data, 'err': 0})

    def _q_lookup(self, component):
        if not (is_url_signed() or (
                get_request().user and get_request().user.can_go_in_admin())):
            raise AccessForbiddenError()
        return ApiUserDirectory(get_publisher().user_class.get(component))


class ApiDirectory(Directory):
    _q_exports = ['forms', 'roles', ('reverse-geocoding', 'reverse_geocoding'),
            'formdefs', 'categories', 'user', 'users']

    forms = ApiFormsDirectory()
    formdefs = ApiFormdefsDirectory()
    categories = ApiCategoriesDirectory()
    user = ApiUserDirectory()
    users = ApiUsersDirectory()

    def reverse_geocoding(self):
        try:
            lat = get_request().form['lat']
            lon = get_request().form['lon']
        except KeyError:
            raise QueryError
        nominatim_url = get_publisher().get_site_option('nominatim_url')
        if not nominatim_url:
            nominatim_url = 'http://nominatim.openstreetmap.org'
        get_response().set_content_type('application/json')
        return urllib2.urlopen('%s/reverse?format=json&zoom=18&addressdetails=1&lat=%s&lon=%s' % (
            nominatim_url, lat, lon)).read()

    def roles(self):
        get_response().set_content_type('application/json')
        if not (is_url_signed() or (
                get_request().user and get_request().user.can_go_in_admin())):
            raise AccessForbiddenError()
        list_roles = []
        charset = get_publisher().site_charset
        for role in Role.select():
            list_roles.append(role.get_json_export_dict())
        get_response().set_content_type('application/json')
        return json.dumps({'data': list_roles})