diff --git a/setup.py b/setup.py index 92704a7..317cd22 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools.command.sdist import sdist class eo_sdist(sdist): def run(self): - print "creating VERSION file" + print("creating VERSION file") if os.path.exists('VERSION'): os.remove('VERSION') version = get_version() @@ -17,7 +17,7 @@ class eo_sdist(sdist): version_file.write(version) version_file.close() sdist.run(self) - print "removing VERSION file" + print("removing VERSION file") if os.path.exists('VERSION'): os.remove('VERSION') @@ -38,7 +38,7 @@ def get_version(): else: result = '0.0.0-%s' % len(subprocess.check_output( ['git', 'rev-list', 'HEAD']).splitlines()) - return result.replace('-', '.').replace('.g', '+g') + return result.decode('utf-8').replace('-', '.').replace('.g', '+g') return '0.0.0' diff --git a/src/ldaptools/ldap_source.py b/src/ldaptools/ldap_source.py index 9fd85eb..43f5ca3 100644 --- a/src/ldaptools/ldap_source.py +++ b/src/ldaptools/ldap_source.py @@ -25,7 +25,7 @@ class LDAPSource(object): continue entry = idict(entry) if 'objectclass' in entry: - entry['objectclass'] = [istr(v) for v in entry['objectclass']] + entry['objectclass'] = [istr(v.decode('utf-8')) for v in entry['objectclass']] yield dn, entry def __iter__(self): diff --git a/src/ldaptools/ldapsync/cmd.py b/src/ldaptools/ldapsync/cmd.py index e1e62d0..c26b834 100644 --- a/src/ldaptools/ldapsync/cmd.py +++ b/src/ldaptools/ldapsync/cmd.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import argparse import sys @@ -18,16 +20,16 @@ def or_type(f1, f2): def f(value): try: return f1(value) - except argparse.ArgumentTypeError, e1: + except argparse.ArgumentTypeError as e1: try: return f2(value) - except argparse.ArgumentTypeError, e2: + except argparse.ArgumentTypeError as e2: raise argparse.ArgumentTypeError('%s and %s' % (e1.args[0], e2.args[0])) return f def object_class_pivot(value): - value = filter(None, map(str.strip, map(str.lower, value.split()))) + value = list(filter(None, map(str.strip, map(str.lower, value.split())))) if len(value) != 2: raise argparse.ArgumentTypeError('%r is not a pair of an objectClass and an attribute name') return value @@ -100,18 +102,18 @@ Base DN of the source is remapped to another DN in the target directory''') attributes = list(attributes) if not attributes: parser.print_help() - print 'Yout must give at least one attribute to synchronize' + print('You must give at least one attribute to synchronize') if options.verbose: - print 'Synchronizing ', + print('Synchronizing', end=' ') if hasattr(options.source_uri, 'read'): if options.verbose: - print options.source_uri.name, + print(options.source_uri.name, end=' ') source = ldif_utils.ListLDIFParser(options.source_uri) source.parse() else: if options.verbose: - print options.source_uri, + print(options.source_uri, end=' ') conn = paged.PagedLDAPObject(options.source_uri) if options.source_uri.startswith('ldapi://'): conn.sasl_interactive_bind_s("", ldap.sasl.external()) @@ -122,7 +124,7 @@ Base DN of the source is remapped to another DN in the target directory''') filterstr=options.source_filter) if options.verbose: - print 'to', options.target_uri + print('to', options.target_uri, end=' ') target_conn = paged.PagedLDAPObject(options.target_uri) if options.target_uri.startswith('ldapi://'): target_conn.sasl_interactive_bind_s("", ldap.sasl.external()) @@ -142,16 +144,16 @@ Base DN of the source is remapped to another DN in the target directory''') synchronize.build_actions() if options.verbose: for action in synchronize.actions: - print ' -', action + print(' -', action) if not synchronize.actions: - print 'Nothing to do.' + print('Nothing to do.') if not options.fake: synchronize.apply_actions() failed_actions = [action for action in synchronize.actions if action.errors] if failed_actions: - print >>sys.stderr, 'Some actions failed:' + print('Some actions failed:', file=sys.stderr) for action in failed_actions: - print ' -', action + print(' -', action) for error in action.errors: - print ' *', error + print(' *', error) raise SystemExit(1) diff --git a/src/ldaptools/ldif_utils.py b/src/ldaptools/ldif_utils.py index 37fbe8b..714fa08 100644 --- a/src/ldaptools/ldif_utils.py +++ b/src/ldaptools/ldif_utils.py @@ -2,8 +2,7 @@ import ldap import ldif from ldap.dn import dn2str -from ldaptools.utils import idict, str2dn - +from ldaptools.utils import idict, str2dn, bytes2str_entry, str2bytes_entry class AddError(Exception): pass @@ -18,13 +17,13 @@ class ListLDIFParser(ldif.LDIFParser): dn = str2dn(dn) dn = [[(part[0].lower(),) + part[1:] for part in rdn] for rdn in dn] dn = dn2str(dn) - self.entries.append((dn, entry)) + self.entries.append((dn, bytes2str_entry(entry))) def add(self, conn): for dn, entry in self.entries: try: - conn.add_s(dn, ldap.modlist.addModlist(entry)) - except Exception, e: + conn.add_s(dn, ldap.modlist.addModlist(str2bytes_entry(entry))) + except Exception as e: raise AddError('error when adding %s' % dn, e) def __iter__(self): diff --git a/src/ldaptools/slapd.py b/src/ldaptools/slapd.py index a18f54c..53247f8 100644 --- a/src/ldaptools/slapd.py +++ b/src/ldaptools/slapd.py @@ -1,3 +1,4 @@ +import codecs import time import tempfile import shutil @@ -6,7 +7,10 @@ import os import ldap import ldap.modlist import ldap.sasl -import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO import atexit from ldaptools.ldif_utils import ListLDIFParser @@ -87,8 +91,9 @@ olcAccess: {{0}}to * process = None schemas = ['core', 'cosine', 'inetorgperson', 'nis', 'eduorg-200210-openldap', 'eduperson', 'supann-2009'] - schemas_ldif = [open(os.path.join(os.path.dirname(__file__), - 'schemas', '%s.ldif' % schema)).read() for schema in schemas] + schemas_ldif = [codecs.open(os.path.join(os.path.dirname(__file__), + 'schemas', '%s.ldif' % schema), + encoding='utf-8').read() for schema in schemas] checkpoints = None data_dirs = None db_index = 1 @@ -172,7 +177,7 @@ olcAccess: {{0}}to * by * manage if context: ldif = ldif.format(**context) slapadd = self.create_process([SLAPADD_PATH, '-v', '-n%d' % db, '-F', self.config_dir]) - stdout, stderr = slapadd.communicate(input=ldif) + stdout, stderr = slapadd.communicate(input=bytearray(ldif, 'utf-8')) assert slapadd.returncode == 0, 'slapadd failed: %s' % stderr def start(self): @@ -266,7 +271,7 @@ olcAccess: {{0}}to * by * manage if context: ldif = ldif.format(**context) - parser = ListLDIFParser(StringIO.StringIO(ldif)) + parser = ListLDIFParser(StringIO(ldif)) parser.parse() conn = self.get_connection_admin() parser.add(conn) diff --git a/src/ldaptools/synchronize.py b/src/ldaptools/synchronize.py index ed9f601..2d573b7 100644 --- a/src/ldaptools/synchronize.py +++ b/src/ldaptools/synchronize.py @@ -8,7 +8,8 @@ import ldap.modlist import ldap.dn -from .utils import batch_generator, to_dict_of_set, idict, str2dn, istr +from .utils import batch_generator, to_dict_of_set, idict, str2dn, istr, \ + bytes2str_entry, str2bytes_entry @functools.total_ordering @@ -48,7 +49,7 @@ class Action(object): for msgid in self.msgids: try: self.results.append(conn.result2(msgid)) - except ldap.LDAPError, e: + except ldap.LDAPError as e: self.errors.append(e) def __str__(self): @@ -62,7 +63,7 @@ class Create(Action): order = 3 def do(self, conn): - self.msgids.append(conn.add(self.dn, ldap.modlist.addModlist(self.entry))) + self.msgids.append(conn.add(self.dn, ldap.modlist.addModlist(str2bytes_entry(self.entry)))) class Rename(Action): @@ -83,7 +84,7 @@ class Update(Action): def do(self, conn): modlist = [] - for key, values in self.entry.iteritems(): + for key, values in str2bytes_entry(self.entry).items(): modlist.append((ldap.MOD_REPLACE, key, values)) self.msgids.append(conn.modify(self.dn, modlist)) @@ -144,30 +145,39 @@ class Synchronize(object): def get_pivot_attribute(self, dn, entry): '''Find a pivot attribute value for an LDAP entry''' for objc, attr in self.pivot_attributes: - entry['objectclass'] = map(istr, entry['objectclass']) - if objc in entry['objectclass']: + if istr(objc) in [istr(oc.decode('utf-8')) + if isinstance(oc, bytes) else oc + for oc in entry['objectclass']]: try: value = entry[attr] except KeyError: raise Exception('entry %s missing pivot attribute %s: %s' % (dn, attr, entry)) break else: - raise Exception('entry %s has unknown objectclasses %s' % (dn, entry['objectclass'])) + raise Exception('entry %s has unknown objectclasses %s' % (dn, + [objclass for objclass in entry['objectclass']])) if len(value) != 1: raise Exception('entry %s pivot attribute %s must have only one value' % (dn, attr)) + value = value[0] + """ + may be used for input entries or output entries. + decoding may be required + """ + if isinstance(value, bytes): + value = value.decode('utf-8') if attr in self.case_insensitive_attribute: - value = map(istr, value) - return objc, attr, value[0] + value = istr(value) + return objc, attr, value def get_target_entries(self, filterstr=None, attributes=[]): '''Return all target entries''' try: # Check base DN exist self.target_conn.search_s(self.target_dn, ldap.SCOPE_BASE) - l = self.target_conn.paged_search_ext_s(self.target_dn, ldap.SCOPE_SUBTREE, + res = self.target_conn.paged_search_ext_s(self.target_dn, ldap.SCOPE_SUBTREE, filterstr=filterstr or self.all_filter, attrlist=attributes) - return ((dn, idict(entry)) for dn, entry in l if dn) + return ((dn, idict(bytes2str_entry(entry))) for dn, entry in res if dn) except ldap.NO_SUCH_OBJECT: return [] @@ -215,7 +225,7 @@ class Synchronize(object): seen_dn.add(out_dn) self.rename(new_out_dn, target_dn) renamed_dn[str2dn(new_out_dn)] = str2dn(target_dn) - if to_dict_of_set(out_entry) != to_dict_of_set(entry): + if to_dict_of_set(out_entry) != to_dict_of_set(bytes2str_entry(entry)): new_entry = {} for attribute in self.attributes: if attribute in to_dict_of_set(entry): @@ -233,10 +243,10 @@ class Synchronize(object): self.actions = [] # Order source entries by DN depth entries = list(self.source) - entries.sort(key=lambda (dn, entry): len(str2dn(dn))) + entries.sort(key=lambda dn_entry: len(str2dn(dn_entry[0]))) for dn, entry in entries: for key in entry.keys(): - if not key in self.attributes: + if not str(key.lower()) in self.attributes: del entry[key] # First create, rename and update for batch in batch_generator(entries, self.BATCH_SIZE): diff --git a/src/ldaptools/utils.py b/src/ldaptools/utils.py index cf4a2e1..c3c5370 100644 --- a/src/ldaptools/utils.py +++ b/src/ldaptools/utils.py @@ -1,4 +1,5 @@ import ldap.dn +from six import string_types # Copied from http://code.activestate.com/recipes/194371-case-insensitive-strings/ @@ -7,7 +8,7 @@ class istr(str): Performs like str except comparisons are case insensitive.""" def __init__(self, strMe): - str.__init__(self, strMe) + super(str, self).__init__() self.__lowerCaseMe = strMe.lower() def __repr__(self): @@ -80,7 +81,7 @@ class idict(dict): def findkey(self, item): """A caseless way of checking if a key exists or not. It returns None or the correct key.""" - if not isinstance(item, str): + if not isinstance(item, string_types): raise TypeError('Keywords for this object must be strings. You supplied %s' % type(item)) key = item.lower() try: @@ -147,13 +148,13 @@ class idict(dict): def has_key(self, item): """A case insensitive test for keys.""" - if not isinstance(item, str): + if not isinstance(item, string_types): return False # should never have a non-string key return item.lower() in self._keydict # does the key exist def __contains__(self, item): """A case insensitive __contains__.""" - if not isinstance(item, str): + if not isinstance(item, string_types): return False # should never have a non-string key return item.lower() in self._keydict # does the key exist @@ -212,7 +213,7 @@ class idict(dict): return True def __ne__(self, other): - return not (self == other) + return not (self == other) def batch_generator(gen, *batch_size): @@ -229,7 +230,7 @@ def batch_generator(gen, *batch_size): def to_dict_of_set(d): - r = idict({k: set(v) for k, v in d.iteritems()}) + r = idict({k: set(v) for k, v in d.items()}) if 'objectclass' in r: r['objectclass'] = set(istr(v) for v in r['objectclass']) return r @@ -237,3 +238,17 @@ def to_dict_of_set(d): def str2dn(s): return tuple(map(tuple, ldap.dn.str2dn(s))) + + +def bytes2str_entry(entry): + str_entry = {} + for key, values in entry.items(): + str_entry[key] = [v.decode('utf-8') if isinstance(v, bytes) else v for v in values] + return str_entry + +def str2bytes_entry(entry): + bytes_entry = {} + + for key, values in entry.items(): + bytes_entry[key] = [v.encode('utf-8') if isinstance(v, string_types) else v for v in values] + return bytes_entry diff --git a/tests/conftest.py b/tests/conftest.py index eb6f012..d400361 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import pytest import tempfile import os @@ -97,7 +99,7 @@ def attributes_path(request, attributes): handle, path = tempfile.mkstemp() with open(path, 'w') as f: for attribute in attributes: - print >>f, ' %s ' % attribute + print(' %s ' % attribute, file=f) f.flush() def finalize(): os.unlink(path) diff --git a/tests/test_ldif_utils.py b/tests/test_ldif_utils.py index ad1b961..0ed157c 100644 --- a/tests/test_ldif_utils.py +++ b/tests/test_ldif_utils.py @@ -1,10 +1,13 @@ -import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO from ldaptools.ldif_utils import ListLDIFParser def test_ldifparser(): - parser = ListLDIFParser(StringIO.StringIO('''dn: o=orga + parser = ListLDIFParser(StringIO('''dn: o=orga objectClass: organization ''')) diff --git a/tests/test_slapd.py b/tests/test_slapd.py index e4888da..ef89b37 100644 --- a/tests/test_slapd.py +++ b/tests/test_slapd.py @@ -36,8 +36,8 @@ def test_any(any_slapd): def test_ssl_client_cert(slapd_ssl): conn = slapd_ssl.get_connection_admin() conn.modify_s('cn=config', [ - (ldap.MOD_ADD, 'olcTLSCACertificateFile', slapd_ssl.tls[1]), - (ldap.MOD_ADD, 'olcTLSVerifyClient', 'demand'), + (ldap.MOD_ADD, 'olcTLSCACertificateFile', slapd_ssl.tls[1].encode('utf-8')), + (ldap.MOD_ADD, 'olcTLSVerifyClient', b'demand'), ]) with pytest.raises((ldap.SERVER_DOWN, ldap.CONNECT_ERROR)): diff --git a/tests/test_synchronize.py b/tests/test_synchronize.py index a1d1f69..292b56a 100644 --- a/tests/test_synchronize.py +++ b/tests/test_synchronize.py @@ -1,4 +1,7 @@ -import StringIO +try: + from StringIO import StringIO +except ImportError: + from io import StringIO import ldap @@ -17,7 +20,7 @@ def test_synchronize_ldif(slapd): conn = slapd.get_connection_admin() def syn_ldif(ldif): - parser = ListLDIFParser(StringIO.StringIO(ldif)) + parser = ListLDIFParser(StringIO(ldif)) parser.parse() synchronize = Synchronize(parser, 'o=orga', conn, 'o=orga', pivot_attributes=pivot_attributes, @@ -163,7 +166,7 @@ def test_synchronize_deep_rename(slapd): conn = slapd.get_connection_admin() def syn_ldif(ldif): - parser = ListLDIFParser(StringIO.StringIO(ldif)) + parser = ListLDIFParser(StringIO(ldif)) parser.parse() synchronize = Synchronize(parser, 'o=orga', conn, 'o=orga', pivot_attributes=pivot_attributes, diff --git a/tox.ini b/tox.ini index 063b256..fec2f88 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/ldaptools/ -envlist = coverage,package +envlist = py2-coverage-ldap2,py{2,3}-coverage-ldap3,package [testenv] usedevelop = true @@ -16,7 +16,8 @@ deps = pytest pytest-cov pytest-random - python-ldap<3 + ldap3: python-ldap>2 + ldap2: python-ldap<3 commands = py.test {env:COVERAGE:} {posargs:--random tests}