From 97b3a59326f57cebcccd24d4db14de2fc58816e4 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Sun, 8 Feb 2015 23:15:58 +0100 Subject: [PATCH] Ajout d'une commande metasync pour la synchronisation d'un annuaire distant --- lib/metasync | 378 ++++++++++++++++++++++++++++++++++++++++++++++ lib/metasync.help | 1 + 2 files changed, 379 insertions(+) create mode 100755 lib/metasync create mode 100644 lib/metasync.help diff --git a/lib/metasync b/lib/metasync new file mode 100755 index 0000000..ca095c1 --- /dev/null +++ b/lib/metasync @@ -0,0 +1,378 @@ +#!/usr/bin/env python +import argparse + +import ldap +import ldap.modlist +from ldap.ldapobject import ReconnectLDAPObject +import ldap.sasl +from ldap.controls import SimplePagedResultsControl +from ldap.filter import filter_format + + +attributes = [ + 'audio', + 'businessCategory', + 'carLicense', + 'cn', + 'dc', + 'departmentNumber', + 'description', + 'destinationIndicator', + 'displayName', + 'eduOrgHomePageURI', + 'eduOrgIdentityAuthNPolicyURI', + 'eduOrgLegalName', + 'eduOrgSuperiorURI', + 'eduOrgWhitePagesURI', + 'eduPersonAffiliation', + 'eduPersonAssurance', + 'eduPersonEntitlement', + 'eduPersonNickname', + 'eduPersonOrgDN', + 'eduPersonOrgUnitDN', + 'eduPersonPrimaryAffiliation', + 'eduPersonPrimaryOrgUnitDN', + 'eduPersonPrincipalName', + 'eduPersonScopedAffiliation', + 'eduPersonTargetedID', + 'employeeNumber', + 'employeeType', + 'facsimileTelephoneNumber', + 'givenName', + 'homePhone', + 'homePostalAddress', + '# http', + 'initials', + 'internationaliSDNNumber', + 'jpegPhoto', + 'l', + 'labeledURI', + 'mail', + 'mailForwardingAddress', + 'manager', + 'member', + 'mobile', + 'o', + 'ou', + 'owner', + 'pager', + 'photo', + 'physicalDeliveryOfficeName', + 'postalAddress', + 'postalCode', + 'postOfficeBox', + 'preferredDeliveryMethod', + 'preferredLanguage', + 'registeredAddress', + 'roomNumber', + 'searchGuide', + 'secretary', + 'seeAlso', + 'sn', + 'st', + 'street', + 'supannActivite', + 'supannAffectation', + 'supannAliasLogin', + 'supannAutreMail', + 'supannAutreTelephone', + 'supannCivilite', + 'supannCodeEntite', + 'supannCodeEntiteParent', + 'supannCodeINE', + 'supannEmpCorps', + 'supannEmpId', + 'supannEntiteAffectation', + 'supannEntiteAffectationPrincipale', + 'supannEtablissement', + 'supannEtuAnneeInscription', + 'supannEtuCursusAnnee', + 'supannEtuDiplome', + 'supannEtuElementPedagogique', + 'supannEtuEtape', + 'supannEtuId', + 'supannEtuInscription', + 'supannEtuRegimeInscription', + 'supannEtuSecteurDisciplinaire', + 'supannEtuTypeDiplome', + 'supannGroupeAdminDN', + 'supannGroupeDateFin', + 'supannGroupeLecteurDN', + 'supannListeRouge', + 'supannMailPerso', + 'supannOrganisme', + 'supannParrainDN', + 'supannRefId', + 'supannRole', + 'supannRoleEntite', + 'supannRoleGenerique', + 'supannTypeEntite', + 'supannTypeEntiteAffectation', + 'telephoneNumber', + 'teletexTerminalIdentifier', + 'telexNumber', + 'title', + 'uid', + 'userCertificate', + 'userPassword', + 'userPKCS12', + 'userSMIMECertificate', + 'x121Address', + 'x500UniqueIdentifier', + 'objectClass', +] + +class PagedResultsSearchObject: + page_size = 500 + + def paged_search_ext_s(self,base,scope,filterstr='(objectClass=*)',attrlist=None,attrsonly=0,serverctrls=None,clientctrls=None,timeout=-1,sizelimit=0): + """ + Behaves exactly like LDAPObject.search_ext_s() but internally uses the + simple paged results control to retrieve search results in chunks. + + This is non-sense for really large results sets which you would like + to process one-by-one + """ + + while True: # loop for reconnecting if necessary + + req_ctrl = SimplePagedResultsControl(True,size=self.page_size,cookie='') + + try: + + # Send first search request + msgid = self.search_ext( + base, + scope, + filterstr=filterstr, + attrlist=attrlist, + attrsonly=attrsonly, + serverctrls=(serverctrls or [])+[req_ctrl], + clientctrls=clientctrls, + timeout=timeout, + sizelimit=sizelimit + ) + + all_results = [] + + while True: + rtype, rdata, rmsgid, rctrls = self.result3(msgid) + for result in rdata: + yield result + all_results.extend(rdata) + # Extract the simple paged results response control + pctrls = [ + c + for c in rctrls + if c.controlType == SimplePagedResultsControl.controlType + ] + if pctrls: + if pctrls[0].cookie: + # Copy cookie from response control to request control + req_ctrl.cookie = pctrls[0].cookie + msgid = self.search_ext( + base, + scope, + filterstr=filterstr, + attrlist=attrlist, + attrsonly=attrsonly, + serverctrls=(serverctrls or [])+[req_ctrl], + clientctrls=clientctrls, + timeout=timeout, + sizelimit=sizelimit + ) + else: + break # no more pages available + + except ldap.SERVER_DOWN,e: + self.reconnect(self._uri) + else: + break + + +class PagedLDAPObject(ReconnectLDAPObject,PagedResultsSearchObject): + pass + +lconn = PagedLDAPObject('ldapi://', retry_max=100, retry_delay=2) +lconn.sasl_interactive_bind_s("", ldap.sasl.external()) + +get_all_filter = '(|(objectclass=dcobject)(objectclass=organizationalUnit)(objectclass=supannPerson)(objectclass=supannGroupe)(objectclass=supannEntite))' +pivot_attributes = [('supannPerson', 'uid'), + ('supannGroupe', 'cn'), + ('supannEntite', 'supanncodeentite'), + ('dcObject', 'dc'), + ('organizationalUnit', 'ou')] +pivot_attrs = ['objectclass', 'uid', 'cn', 'supanncodeentite', 'dc', 'ou'] + +parser = argparse.ArgumentParser(description='Synchronize a supann directory with a meta-directory copy') +parser.add_argument('ldap_uri', help='URL of the other LDAP to synchronize locally') +parser.add_argument('ldap_newbasedn', help='new basedn for entries copied locally (ex.: dc=univ-test,ou=meta)') +parser.add_argument('ldap_basedn', help='basedn of the remote supann directory (ex.: dc=univ-test,dc=fr)') +parser.add_argument('ldap_binddn', nargs='?', help='bind DN to read the remote directory', default=None) +parser.add_argument('ldap_bindpwd', nargs='?', help='bind password to read the remote directory') +parser.add_argument('--quiet', dest='verbose', action='store_false', default=True, help='do not report all modifications') +parser.add_argument('--fake', action='store_true', default=False, help='do not apply modifications, only simulate them') +args = parser.parse_args() + +if args.verbose: + print 'Synchronizing LDAP directory at', args.ldap_binddn, 'locally.' + print 'BaseDN:', args.ldap_basedn + if args.ldap_binddn: + print 'BindDN:', args.ldap_binddn + print 'BindPWD:', args.ldap_bindpwd + +rconn = PagedLDAPObject(args.ldap_uri, retry_max=100, retry_delay=2) +if args.ldap_binddn: + rconn.simple_bind_s(args.ldap_binddn, args.ldap_bindpwd) + +def lower_keys(d): + return dict((key.lower(), value) for key, value in d.iteritems()) + +def batch_generator(gen, batch_size): + batch = [] + for result in gen: + batch.append(result) + if len(batch) == batch_size: + yield batch + batch = [] + if len(batch): + yield batch + +def get_pivotattr(dn, entry): + '''Find a pivot attribute value for an LDAP entry''' + for objc, attr in pivot_attributes: + if objc in entry['objectclass']: + try: + value = entry[attr] + except KeyError: + raise Exception('entry %s is missing pivot attribute %s: %s' % (dn, attr, entry)) + break + else: + raise Exception('entry %s has unknown objectclasses %s' % (dn, entry['objectclass'])) + if len(value) != 1: + raise Exception('error on entry %s: pivot attribute %s must have only one value' % (dn, attr)) + return objc, attr, value[0] + +def get_found_dns(filters): + '''Return a mapping from pivot attributes to dns''' + or_filter = '(|%s)' % ''.join(filters) + found = {} + try: + for dn, entry in lconn.search_s(args.ldap_newbasedn, ldap.SCOPE_SUBTREE, or_filter, attributes): + entry = lower_keys(entry) + objc, attr, value = get_pivotattr(dn, entry) + found[(attr, value)] = dn, entry + except ldap.NO_SUCH_OBJECT: + pass + return found + +CREATE = 1 +UPDATE = 2 +RENAME = 3 +DELETE = 4 + +def new_dn(old_dn): + return old_dn[:-len(args.ldap_basedn)] + args.ldap_newbasedn + +def to_dict_of_set(d): + return {k: set(v) for k, v in d.iteritems()} + +# Create & Update +batched = 0 +batch = [] +seen_dns = set() +renames, creates, updates, deletes = 0, 0, 0, 0 +for results in batch_generator(rconn.paged_search_ext_s( + args.ldap_basedn, + ldap.SCOPE_SUBTREE, + get_all_filter, + attributes), 200): + new_dns = dict() + filters = [] + for dn, entry in results: + entry = lower_keys(entry) + objc, attr, value = get_pivotattr(dn, entry) + new_dns[(attr, value)] = dn, entry + filter_tpl = '(&(objectclass=%%s)(%s=%%s))' % attr + filters.append(filter_format( + filter_tpl, (objc, value))) + found_dns = get_found_dns(filters) + actions = [] + for k, (dn, entry) in new_dns.iteritems(): + target_dn = new_dn(dn) + seen_dns.add(target_dn) + if k in found_dns: + existing_dn, existing_entry = found_dns[k] + if existing_dn != target_dn: + actions.append((RENAME, found_dns[k], target_dn)) + renames += 1 + if to_dict_of_set(existing_entry) != to_dict_of_set(entry): + actions.append((UPDATE, target_dn, entry)) + updates += 1 + else: + actions.append((CREATE, target_dn, entry)) + creates += 1 + +try: + for dn, entry in lconn.paged_search_ext_s( + args.ldap_newbasedn, + ldap.SCOPE_SUBTREE, + get_all_filter): + if dn not in seen_dns: + actions.append((DELETE, dn)) + deletes += 1 +except ldap.NO_SUCH_OBJECT: + pass + +printing = { + CREATE: ('- Create', 1), + UPDATE: ('- Update', 1), + RENAME: ('- Rename', 1, 'to', 2), + DELETE: ('- Delete', 1), +} + + + +def print_action(action): + for do in printing[action[0]]: + if isinstance(do, int): + print action[do], + else: + print do, + print + +def apply_action(action): + t = action[0] + if t == CREATE: + dn, entry = action[1:] + lconn.add(dn, ldap.modlist.addModlist(entry)) + if t == RENAME: + old_dn, new_dn = action[1:] + lconn.modrdn(old_dn, new_dn) + if t == UPDATE: + dn, entry = action[1:] + modlist = [] + for key, values in entry.iteritems(): + modlist.append((ldap.MOD_REPLACE, key, values)) + lconn.modify(dn, modlist) + if t == DELETE: + lconn.delete(action[1]) + +actions.sort(key=lambda x: (x[0] == DELETE, len(x[1]))) +if args.verbose: + print 'Actions:' +for action in actions: + if args.verbose: + print_action(action) + if not args.fake: + apply_action(action) + else: + print 'Fake, doing nothing' + +if args.verbose: + print 'Waiting for completion..', +i = len(actions) +while i: + lconn.result() + i -= 1 +print ' done' diff --git a/lib/metasync.help b/lib/metasync.help new file mode 100644 index 0000000..43ddf28 --- /dev/null +++ b/lib/metasync.help @@ -0,0 +1 @@ +synchronise un annuaire distant dans le méta-annuaire local