backends: complete rewrite of the interface

The old interface was to specific for sqlalchemy this new one allow to
write new backends

WARNING: this commit could break compability for some filter which uses
the old interface

mandaye/auth/authform.py

@@ -24,7 +24,7 @@ from mandaye.response import _500, _302, _401
 from mandaye.response import template_response
 from mandaye.server import get_response
 
-from mandaye.backends.default import backend
+from mandaye.backends.default import Association
 
 try:
     from Crypto.Cipher import AES
@@ -203,17 +203,11 @@ a password_field key if you want to encode a password.")
         if config.encrypt_sp_password:
             password = self.encrypt_pwd(post_values[self.form_values['password_field']])
             post_values[self.form_values['password_field']] = password
-        service_provider = backend.ManagerServiceProvider.get_or_create(self.site_name)
+
-        idp_user = backend.ManagerIDPUser.get_or_create(unique_id)
+        asso_id = Association.update_or_create(self.site_name, sp_login,
-        sp_user = backend.ManagerSPUser.get(sp_login, idp_user, service_provider)
+                post_values, unique_id)
-        if sp_user:
-            sp_user.post_values = post_values
-            backend.ManagerSPUser.save()
-        else:
-            sp_user = backend.ManagerSPUser.create(sp_login, post_values,
-                    idp_user, service_provider)
         env['beaker.session']['unique_id'] = unique_id
-        env['beaker.session'][self.site_name] = sp_user.id
+        env['beaker.session'][self.site_name] = asso_id
         env['beaker.session'].save()
 
     def associate_submit(self, env, values, request, response):
@@ -253,22 +247,21 @@ a password_field key if you want to encode a password.")
             qs['type'] = 'badlogin'
             return _302(self.urls.get('associate_url') + "?%s" % urllib.urlencode(qs))
 
-    def _login_sp_user(self, sp_user, env, condition, values):
+    def _login_sp_user(self, association, env, condition, values):
         """ Log in sp user
         """
-        if not sp_user.login:
+        if not association['sp_login']:
             return _500(env['PATH_INFO'],
                     'Invalid values for AuthFormDispatcher.login')
-        post_values = copy.copy(sp_user.post_values)
+        post_values = copy.copy(association['sp_post_values'])
         if config.encrypt_sp_password:
             password = self.decrypt_pwd(post_values[self.form_values['password_field']])
             post_values[self.form_values['password_field']] = password
         response = self.replay(env, post_values)
         qs = parse_qs(env['QUERY_STRING'])
         if condition and eval(condition):
-            sp_user.last_connection = datetime.now()
+            Association.update_last_connection(association['id'])
-            backend.ManagerSPUser.save()
+            env['beaker.session'][self.site_name] = association['id']
-            env['beaker.session'][self.site_name] = sp_user.id
             env['beaker.session'].save()
             if qs.has_key('next_url'):
                 return _302(qs['next_url'][0], response.cookies)
@@ -295,13 +288,11 @@ a password_field key if you want to encode a password.")
 
         logger.debug('User %s successfully login' % env['beaker.session']['unique_id'])
 
-        idp_user = backend.ManagerIDPUser.get_or_create(unique_id)
+        association = Association.get_last_connected(self.site_name, unique_id)
-        service_provider = backend.ManagerServiceProvider.get_or_create(self.site_name)
+        if not association:
-        sp_user = backend.ManagerSPUser.get_last_connected(idp_user, service_provider)
-        if not sp_user:
             logger.debug('User %s is not associate' % env['beaker.session']['unique_id'])
             return _302(self.urls.get('associate_url') + "?type=first")
-        return self._login_sp_user(sp_user, env, values['condition'], values)
+        return self._login_sp_user(association, env, values['condition'], values)
 
     def logout(self, env, values, request, response):
         """ Destroy the Beaker session
@@ -348,15 +339,13 @@ a password_field key if you want to encode a password.")
         if not qs.has_key('id') and not unique_id:
             return _401('Access denied: beaker session invalid or not qs id')
         if qs.has_key('id'):
-            id = qs['id'][0]
+            asso_id = qs['id'][0]
-            sp_user = backend.ManagerSPUser.get_by_id(id)
+            association = Association.get_by_id(asso_id)
         else:
-            service_provider = backend.ManagerServiceProvider.get(self.site_name)
+            association = Association.get_last_connected(self.site_name, unique_id)
-            idp_user = backend.ManagerIDPUser.get(unique_id)
+        if not association:
-            sp_user = backend.ManagerSPUser.get_last_connected(idp_user, service_provider)
-        if not sp_user:
             return _302(self.urls.get('associate_url'))
-        return self._login_sp_user(sp_user, env, 'response.code==302', values)
+        return self._login_sp_user(association, env, 'response.code==302', values)
 
     def disassociate(self, env, values, request, response):
         """ Disassociate an account with the Mandaye account
@@ -376,20 +365,19 @@ a password_field key if you want to encode a password.")
         if qs.has_key('next_url'):
             next_url = qs['next_url'][0]
         if qs.has_key('id'):
-            sp_id = qs['id'][0]
+            asso_id = qs['id'][0]
-            sp_user = backend.ManagerSPUser.get_by_id(sp_id)
+            if Association.has_id(asso_id):
-            if sp_user:
+                Association.delete(asso_id)
-                backend.ManagerSPUser.delete(sp_user)
+                if Association.get(self.site_name, unique_id):
-                if backend.ManagerSPUser.get_sp_users(unique_id, self.site_name):
                     env['QUERY_STRING'] = ''
                     return self.change_user(env, values, request, response)
             else:
                 return _401('Access denied: bad id')
         elif qs.has_key('sp_name'):
             sp_name = qs['sp_name'][0]
-            for sp_user in \
+            for asso in \
-                    backend.ManagerSPUser.get_sp_users(unique_id, sp_name):
+                    Association.get(sp_name, unique_id):
-                backend.ManagerSPUser.delete(sp_user)
+                Association.delete(asso['id'])
         else:
             return _401('Access denied: no id or sp name')
         values['next_url'] = next_url

mandaye/backends/default.py

@@ -4,80 +4,6 @@ from importlib import import_module
 from mandaye import config
 from mandaye.exceptions import ImproperlyConfigured
 
-class DefaultManagerIDPUser:
-
-    @staticmethod
-    def get(unique_id, idp_id='default'):
-        pass
-
-    @staticmethod
-    def create(unique_id, idp_id='default'):
-        pass
-
-    @staticmethod
-    def get_or_create(unique_id, idp_id='default'):
-        pass
-
-    @staticmethod
-    def delete(idp_user):
-        pass
-
-    @staticmethod
-    def save(idp_user):
-        pass
-
-class DefaultManagerSPUser:
-
-    @staticmethod
-    def get(login, idp_user, service_provider):
-        pass
-
-    @staticmethod
-    def get_by_id(id):
-        pass
-
-    @staticmethod
-    def get_last_connected(idp_user, service_provider):
-        pass
-
-    @staticmethod
-    def create(login, post_values, idp_user, service_provider):
-        pass
-
-    @staticmethod
-    def get_or_create(login, post_values, idp_user, service_provider):
-        pass
-
-    @staticmethod
-    def delete(sp_user):
-        pass
-
-    @staticmethod
-    def save(sp_user):
-        pass
-
-class DefaultServiceProvider:
-
-    @staticmethod
-    def get(name):
-        pass
-
-    @staticmethod
-    def create(name):
-        pass
-
-    @staticmethod
-    def get_or_create(name):
-        pass
-
-    @staticmethod
-    def delete(service_provider):
-        pass
-
-    @staticmethod
-    def save(service_provider):
-        pass
-
 def import_backend(path):
     try:
         mod = import_module(path)
@@ -85,8 +11,79 @@ def import_backend(path):
         raise ImproperlyConfigured('Error importing backend %s: "%s"' % (path, e))
     return mod
 
-backend = import_backend(config.storage_backend)
+storage_conn = None
-ManagerServiceProvider = backend.ManagerServiceProvider
+if config.storage_backend == "mandaye.backends.sql":
-ManagerIDPUser = backend.ManagerIDPUser
+    from sqlalchemy import create_engine
-ManagerSPUser = backend.ManagerSPUser
+    from sqlalchemy.orm import sessionmaker, scoped_session
+    if not "sqlite" in config.db_url:
+       storage_conn = scoped_session(
+                sessionmaker(
+                    bind=create_engine(config.db_url, pool_size=16,
+                        pool_recycle=1800)
+                    )
+                )
+    else:
+        storage_conn = scoped_session(
+                sessionmaker(
+                    bind=create_engine(config.db_url)
+                    )
+                )
 
+backend = import_backend(config.storage_backend)
+Association = backend.Association
+
+class AssociationExample(object):
+    """
+    association dictionnary return by the following methods:
+    {
+        'id': '', # identifier of your association (must be unique)
+        'sp_name': '', # name of the service provider (defined in the mappers)
+        'sp_login': '', # login on the service provider
+        'sp_post_values': '', # the post values for sp login form
+        'idp_unique_id:': '', # the unique identifier of the identity provider (ex.: a saml NameID)
+        'idp_name':  '', # identity provide name
+        'last_connection':  datetime.datetime, # last connection with this association
+        'creation_date':  datetime.datetime, # creation date of this association
+    }
+    """
+
+    @staticmethod
+    def get(sp_name, idp_unique_id, idp_name='dafault'):
+        """ return a list of dict with associations that matching all of this options """
+        pass
+
+    @staticmethod
+    def get_by_id(asso_id):
+        """ return an dict of the association with the id or None if it doesn't exist """
+        pass
+
+    @staticmethod
+    def has_id(asso_id):
+        """ return a boolean """
+        pass
+
+    @staticmethod
+    def update_or_create(sp_name, sp_login, sp_post_values, idp_unique_id, idp_name):
+        """ update or create an associtaion which match the following values
+        return the association id
+        """
+        pass
+
+    @staticmethod
+    def delete(asso_id):
+        """ delete the association which has the following asso_id """
+        pass
+
+    @staticmethod
+    def get_last_connected(sp_name, idp_unique_id, idp_name='default'):
+        """ get the last connecting association which match the parameters
+        return a dict of the association
+        """
+        pass
+
+    @staticmethod
+    def update_last_connection(asso_id):
+        """ update the association last conenction time with the current time
+        return a dict of the association
+        """
+        pass

mandaye/backends/sql.py

@@ -1,162 +1,159 @@
 
+import copy
+
 from datetime import datetime
 
-from mandaye.db import sql_session
+from mandaye.backends.default import storage_conn
 from mandaye.log import logger
 from mandaye.models import IDPUser, SPUser, ServiceProvider
 
-class ManagerIDPUserSQL:
+class Association(object):
+    """
+    association dictionnary return by the following methods:
+    {
+        'id': '', # identifier of your association (must be unique)
+        'sp_name': '', # name of the service provider (defined in the mappers)
+        'sp_login': '', # login on the service provider
+        'sp_post_values': '', # the post values for sp login form
+        'idp_unique_id:': '', # the unique identifier of the identity provider (ex.: a saml NameID)
+        'idp_name':  '', # identity provide name
+        'last_connection':  datetime.datetime, # last connection with this association
+        'creation_date':  datetime.datetime, # creation date of this association
+    }
+    """
 
     @staticmethod
-    def get(unique_id, idp_id='default'):
+    def sp_user2association(sp_user):
-        idp_user = sql_session().query(IDPUser).\
+        return {
-                filter_by(unique_id=unique_id,
+                'id': sp_user.id,
-                        idp_id='default').all()
+                'sp_name': sp_user.service_provider.name,
-        if len(idp_user) > 1:
+                'sp_login': sp_user.login,
-            logger.critical('ManagerIDPUserSQL.get %s not unique' % unique_id)
+                'sp_post_values': sp_user.post_values,
-            raise MandayeException(
+                'idp_unique_id': sp_user.idp_user.unique_id,
-                    'ManagerIDPUserSQL.get : %s is not unique' % unique_id)
+                'idp_name': sp_user.idp_user.idp_id,
-        if idp_user:
+                'last_connection': sp_user.last_connection,
-            return idp_user[0]
+                'creation_date': sp_user.creation_date
-        else:
+                }
-            return None
+
 
     @staticmethod
-    def create(unique_id, idp_id='default'):
+    def get(sp_name, idp_unique_id, idp_name='dafault'):
-        logger.info('Add idp user %s in db' % (unique_id))
+        """ return a list of dict with associations that matching all of this options """
-        idp_user = IDPUser(
+        associations = []
-                    unique_id=unique_id,
+        sp_users = storage_conn.query(SPUser).\
-                    idp_id=idp_id)
-        sql_session().add(idp_user)
-        return idp_user
-
-    @staticmethod
-    def get_or_create(unique_id, idp_id='default'):
-        idp_user= ManagerIDPUserSQL.get(unique_id, idp_id)
-        if idp_user:
-            return idp_user
-        else:
-            return ManagerIDPUserSQL.create(unique_id, idp_id)
-
-    @staticmethod
-    def delete(idp_user):
-        logger.info('Delete in db idp user %s' % idp_user.unique_id)
-        sql_session().delete(idp_user)
-        sql_session().commit()
-
-    @staticmethod
-    def save():
-        sql_session().commit()
-
-class ManagerSPUserSQL:
-
-    @staticmethod
-    def get(login, idp_user, service_provider):
-        return sql_session().query(SPUser).\
                 join(IDPUser).\
                 join(ServiceProvider).\
-                filter(SPUser.login==login).\
+                filter(ServiceProvider.name==sp_name).\
-                filter(SPUser.idp_user==idp_user).\
+                filter(IDPUser.unique_id==idp_unique_id).\
-                filter(SPUser.service_provider==service_provider).\
+                filter(IDPUser.idp_id==idp_name).\
+                all()
+        for sp_user in sp_users:
+            association = Association.sp_user2association(sp_user)
+            associations.append(association)
+        return associations
+
+    @staticmethod
+    def get_by_id(asso_id):
+        """ return an dict of the association with the id or None if it doesn't exist """
+        sp_user = storage_conn.query(SPUser).\
+                filter(SPUser.id==asso_id).first()
+        return Association.sp_user2association(sp_user)
+
+    @staticmethod
+    def has_id(asso_id):
+        """ return a boolean """
+        if storage_conn.query(SPUser).\
+                filter(SPUser.id==asso_id).\
+                count():
+            return True
+        return False
+
+    @staticmethod
+    def update_or_create(sp_name, sp_login, sp_post_values, idp_unique_id, idp_name='default'):
+        """ update or create an associtaion which match the following values
+        return the association id
+        """
+        sp_user = storage_conn.query(SPUser).\
+                join(IDPUser).\
+                join(ServiceProvider).\
+                filter(SPUser.login==sp_login).\
+                filter(ServiceProvider.name==sp_name).\
+                filter(IDPUser.unique_id==idp_unique_id).\
+                filter(IDPUser.idp_id==idp_name).\
                 first()
+        if sp_user:
+            sp_user.post_values = sp_post_values
+            storage_conn.add(sp_user)
+            logger.info('Modify association for %r (%r)', sp_login, idp_unique_id)
+        else:
+            service_provider = storage_conn.query(ServiceProvider).\
+                    filter(ServiceProvider.name==sp_name).first()
+            if not service_provider:
+                logger.info('New service provider %r', sp_name)
+                service_provider = ServiceProvider(name=sp_name)
+                storage_conn.add(service_provider)
+            idp_user = storage_conn.query(IDPUser).\
+                    filter(IDPUser.unique_id==idp_unique_id).\
+                    filter(IDPUser.idp_id==idp_name).\
+                    first()
+            if not idp_user:
+                logger.info('New IDP user %r', idp_unique_id)
+                idp_user = IDPUser(
+                        unique_id=idp_unique_id,
+                        idp_id=idp_name
+                        )
+                storage_conn.add(idp_user)
+            sp_user = SPUser(
+                    login=sp_login,
+                    post_values=sp_post_values,
+                    idp_user=idp_user,
+                    service_provider=service_provider
+                    )
+            storage_conn.add(sp_user)
+            try:
+                storage_conn.commit()
+            except:
+                logger.error("update_or_create transaction failed so we rollback")
+                storage_conn.rollback()
+                raise
+            logger.info('New association %r with %r', sp_login, idp_unique_id)
+        return sp_user.id
 
     @staticmethod
-    def get_by_id(id):
+    def delete(asso_id):
-        return sql_session().query(SPUser).\
+        """ delete the association which has the following asso_id """
-                filter(SPUser.id==id).first()
+        sp_user = storage_conn.query(SPUser).get(asso_id)
+        if not sp_user:
+            logger.warning("Association deletion failed: sp user %r doesn't exist", asso_id)
+            return
+        logger.info("Disassociate account %r (%r)", sp_user.login, sp_user.idp_user.unique_id)
+        storage_conn.delete(sp_user)
 
     @staticmethod
-    def get_last_connected(idp_user, service_provider):
+    def get_last_connected(sp_name, idp_unique_id, idp_name='default'):
-        return sql_session().query(SPUser).\
+        """ get the last connecting association which match the parameters
-                filter(SPUser.idp_user==idp_user).\
+        return a dict of the association
-                filter(SPUser.service_provider==service_provider).\
+        """
+        sp_user = storage_conn.query(SPUser).\
+                filter(ServiceProvider.name==sp_name).\
+                filter(IDPUser.unique_id==idp_unique_id).\
+                filter(IDPUser.idp_id==idp_name).\
                 order_by(SPUser.last_connection.desc()).\
                 first()
-    @staticmethod
-    def get_sp_users(idp_unique_id, service_provider_name):
-        return sql_session().query(SPUser).\
-                    join(IDPUser).\
-                    join(ServiceProvider).\
-                    filter(IDPUser.unique_id==idp_unique_id).\
-                    filter(ServiceProvider.name==service_provider_name).\
-                    order_by(SPUser.last_connection.desc()).\
-                    all()
-
-    @staticmethod
-    def create(login, post_values, idp_user, service_provider):
-        sp_user = SPUser(
-                    login = login,
-                    post_values = post_values,
-                    idp_user = idp_user,
-                    service_provider = service_provider
-                    )
-        logger.info('New association: %s with %s on site %s' % \
-                    (login, idp_user.unique_id, service_provider.name))
-        sql_session().add(sp_user)
-        sql_session().commit()
-        logger.debug('New SP user %s in db' % (login))
-        return sp_user
-
-    @staticmethod
-    def get_or_create(login, post_values, idp_user, service_provider):
-        sp_user = ManagerSPUserSQL.get(login, idp_user, service_provider)
         if sp_user:
-            return sp_user
+            return Association.sp_user2association(sp_user)
-        else:
-            return ManagerSPUserSQL.create(login, post_values,
-                    idp_user, service_provider)
-    @staticmethod
-    def all():
-        return sql_session().query(SPUser).all()
-
-    @staticmethod
-    def delete(sp_user):
-        logger.debug('Disassociate account %s' % sp_user.login)
-        sql_session().delete(sp_user)
-        sql_session().commit()
-
-    @staticmethod
-    def save():
-        sql_session().commit()
-
-class ManagerServiceProviderSQL:
-
-    @staticmethod
-    def get(name):
-        sp = sql_session().query(ServiceProvider).\
-                filter_by(name=name)
-        if sp:
-            return sp.first()
         else:
             return None
 
     @staticmethod
-    def create(name):
+    def update_last_connection(asso_id):
-        logger.info('Add %s service provider into the db' % name)
+        """ update the association last conenction time with the current time
-        sp = ServiceProvider(name=name)
+        return None
-        sql_session().add(sp)
+        """
-        sql_session().commit()
+        sp_user = storage_conn.query(SPUser).get(asso_id)
-        return sp
+        if not sp_user:
-
+            logger.warning("Update last connecting failed: sp user %r doesn't exist", asso_id)
-    @staticmethod
+            return
-    def get_or_create(name):
+        sp_user.last_connection = datetime.now()
-        sp = ManagerServiceProviderSQL.get(name)
+        storage_conn.add(sp_user)
-        if sp:
-            return sp
-        else:
-            return ManagerServiceProviderSQL.create(name)
-
-    @staticmethod
-    def delete(service_provider):
-        logger.debug('Delete service provider %s' % service_provider.name)
-        sql_session().delete(service_provider)
-        sql_session().commit()
-
-    @staticmethod
-    def save():
-        sql_session().commit()
-
-ManagerServiceProvider = ManagerServiceProviderSQL
-ManagerIDPUser = ManagerIDPUserSQL
-ManagerSPUser = ManagerSPUserSQL
 

mandaye/db.py

@@ -1,18 +0,0 @@
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker, scoped_session
-
-from mandaye import config
-
-if not "sqlite" in config.db_url:
-    sql_session = scoped_session(
-            sessionmaker(
-                bind=create_engine(config.db_url, pool_size=16,
-                    pool_recycle=1800)
-                )
-            )
-else:
-    sql_session = scoped_session(
-            sessionmaker(
-                bind=create_engine(config.db_url)
-                )
-            )

mandaye/filters/default.py

@@ -1,7 +1,7 @@
 
 import re
 from mandaye import config
-from mandaye.backends.default import ManagerSPUser
+from mandaye.backends.default import Association
 from mandaye.log import logger
 from mandaye.response import template_response
 
@@ -80,9 +80,9 @@ class MandayeFilter(object):
             values['is_login'] = True
             site_name = env["mandaye.config"]["site_name"]
             if env['beaker.session'].get(site_name):
-                logger.debug('toolbar there is one : %r' % \
+                logger.debug('toolbar there is one : %r' %\
-                        ManagerSPUser.get_by_id(env['beaker.session'].get(site_name)))
+                        env['beaker.session'].get(site_name))
-                current_account = ManagerSPUser.get_by_id(env['beaker.session'].get(site_name))
+                current_account = Association.get_by_id(env['beaker.session'].get(site_name))
             else:
                 logger.debug('toolbar: no account')
         values['account'] = current_account

mandaye/global_config.py

@@ -4,7 +4,7 @@ import os
 _PROJECT_PATH = os.path.join(os.path.dirname(__file__), '..')
 
 # Choose storage
-# Only mandaye.backends.sql at the moment
+# Only sql at the moment
 storage_backend = "mandaye.backends.sql"
 
 ## SQL Backend config
@@ -14,6 +14,7 @@ storage_backend = "mandaye.backends.sql"
 # dialect+driver://username:password@host:port/database
 db_url = 'sqlite:///test.db'
 
+
 # urllib2 debug mode
 debug = False
 

mandaye/server.py

@@ -11,13 +11,13 @@ from md5 import md5
 from urlparse import urlparse
 
 from mandaye import config
-from mandaye.exceptions import ImproperlyConfigured
+from mandaye.backends.default import storage_conn
 from mandaye.dispatcher import Dispatcher
+from mandaye.exceptions import ImproperlyConfigured
 from mandaye.log import logger, UuidFilter
 from mandaye.handlers.default import MandayeRedirectHandler, MandayeErrorHandler
 from mandaye.http import HTTPHeader, HTTPRequest, HTTPResponse
 from mandaye.response import _404, _502, _500
-from mandaye.db import sql_session
 
 
 def get_response(env, request, url, cookiejar=None):
@@ -137,15 +137,18 @@ class MandayeApp(object):
                         response = self.on_request(start_response)
             if not response:
                 response = self.on_response(start_response, _404(env['PATH_INFO']))
-            sql_session.commit()
+            if config.storage_backend == 'mandaye.backends.sql':
+                storage_conn.commit()
         except Exception, e:
-            sql_session.rollback()
+            if config.storage_backend == 'mandaye.backends.sql':
+                storage_conn.rollback()
             if self.raven_client:
                 self.raven_client.captureException()
             response = self.on_response(start_response, _500(env['PATH_INFO'], "Unhandled exception",
                 exception=e, env=env))
         finally:
-            sql_session.close()
+            if config.storage_backend == 'mandaye.backends.sql':
+                storage_conn.close()
         return response
 
     def _get_uuid(self):

mandaye/templates/toolbar.html

@@ -14,7 +14,7 @@
     </li>
     % if account:
     <li>
-      <a href="javascript:mandaye_disassociate_logout('${urls['disassociate_url']}', '${account.login}', ${account.id})" title="Cliquer ici pour supprimer l'association entre ce compte et votre compte citoyen.">Me désassocier</a>
+      <a href="javascript:mandaye_disassociate_logout('${urls['disassociate_url']}', '${account['login']}', ${account['id']})" title="Cliquer ici pour supprimer l'association entre ce compte et votre compte citoyen.">Me désassocier</a>
     </li>
     % endif
     % else:

requirements.txt

@@ -2,7 +2,7 @@ beaker>=1.6
 pycrypto>=2.0
 lxml>=2.0
 xtraceback>=0.3
-sqlalchemy>=0.7,<0.8
+sqlalchemy>=0.7,<1.0
 alembic>=0.5.0
 Mako>=0.4
 python-entrouvert