source: dassldapsync/dassldapsync.py@ 1261

Last change on this file since 1261 was 1261, checked in by joergs, on May 10, 2022 at 10:58:52 AM

extend summary message

  • Property svn:executable set to *
File size: 23.6 KB
Line 
1#!/usr/bin/python
2# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
3
4from __future__ import print_function
5
6# core modules
7import argparse
8try:
9 from configparser import ConfigParser
10except ImportError:
11 from ConfigParser import ConfigParser
12import logging
13from pprint import pprint
14import signal
15import subprocess
16import sys
17import time
18
19# external modules
20import datetime
21import dateutil.parser
22import dateutil.tz
23import ldap
24from ldap.ldapobject import ReconnectLDAPObject
25import ldap.modlist
26from ldap.syncrepl import SyncreplConsumer
27from ldapurl import LDAPUrl
28import ldif
29
30
31
32def getArguments():
33 configfile = '/etc/dassldapsync.conf'
34 parser = argparse.ArgumentParser(description='Synchronize the content of two LDAP servers.')
35 parser.add_argument('-d', '--debug', action='store_true', help="enable debug output")
36 parser.add_argument('configfile', default=configfile,
37 help="Configuration file [default: {}]".format(configfile))
38 return parser.parse_args()
39
40
41class Options(object):
42 def __init__(self):
43 self.delete = True
44 self.starttls = False
45 self.updateonly = False
46 self.filter = None
47 self.attrlist = None
48 self.exclude = None
49 self.renameattr = None
50 self.renamecommand = None
51 self.pwd_max_days = 0
52
53def readLDIFSource(path):
54 logger = logging.getLogger()
55 logger.info("reading LDAP objects from file {}".format(path))
56 with open(path, 'r') as f:
57 parser = ldif.LDIFRecordList(f)
58 parser.parse()
59 result = parser.all_records
60 return result
61
62def readLdapSource(server, binddn, bindpw, basedn, filterstr, attrlist=None, starttls=False):
63 logger = logging.getLogger()
64 logger.info("reading LDAP objects from server {}".format(server))
65 ldapurl = LDAPUrl(hostport="{}:389".format(self.server))
66 con = ldap.initialize(ldapurl. initializeUrl())
67 if starttls:
68 con.start_tls_s()
69 con.simple_bind_s(binddn, bindpw)
70 results = con.search_s(basedn, ldap.SCOPE_SUBTREE, filterstr, attrlist)
71 return results
72
73class LdapSync(object):
74 def __init__(self, destserver,
75 destbinddn, destbindpw,
76 srcbasedn, destbasedn, options=Options()):
77 self.logger = logging.getLogger()
78
79 self.destserver = destserver
80 self.destbasedn = destbasedn
81 self.destbinddn = destbinddn
82 self.destbindpw = destbindpw
83 self.options = options
84
85 self.srcbasedn = srcbasedn
86
87 self.con = None
88
89 self.attrmap = ldap.cidict.cidict({})
90 self.classmap = {}
91
92 #self.junk_objectclasses = [ b"sambaidmapentry" ]
93 #"sambasid",
94 self.junk_objectclasses = []
95 self.junk_attrs = ["authzto",
96 "creatorsname", "createtimestamp", "contextcsn",
97 "entrycsn", "entryuuid",
98 "memberof", "modifiersname", "modifytimestamp",
99 "pwdaccountlockedtime", "pwdchangedtime", "pwdfailuretime",
100 "structuralobjectclass"]
101
102 self.reset_result()
103
104
105 def reset_result(self):
106 self.result = {
107 'unmodified': {'ok': [], 'failed': []},
108 'excluded': {'ok': [], 'failed': []},
109 'add': {'ok': [], 'failed': []},
110 'update': {'ok': [], 'failed': []},
111 'delete': {'ok': [], 'failed': []},
112 }
113
114
115 def _dest_ldap_connect(self):
116 if self.con is None:
117 self.logger.info("connect to destination LDAP server {}".format(self.destserver))
118 ldapurl = LDAPUrl(hostport="{}:389".format(self.destserver))
119 self.con = ldap.initialize(ldapurl. initializeUrl())
120 if self.options.starttls:
121 self.con.start_tls_s()
122 self.con.simple_bind_s(self.destbinddn, self.destbindpw)
123
124 def __adapt_dn(self, dn):
125 # move LDAP object to dest base
126 if self.srcbasedn != self.destbasedn:
127 dn_old = dn
128 rpath = dn[:-len(self.srcbasedn)]
129 dn = rpath+self.destbasedn
130 self.logger.debug("moved {} to {}".format(dn_old, dn))
131 # print "dn:",dn,"src:",srcbasedn,"rpath:",rpath,"dest:",destbasedn
132 return dn
133
134 def __is_dn_included(self, dn):
135 if self.options.exclude is None:
136 return True
137 if dn.lower().endswith(self.options.exclude):
138 return False
139 return True
140
141 def __adapt_source_ldap_objects(self, searchresult):
142 """
143 Do configured modification to the source LDAP objects.
144 """
145 self.logger.debug("modifying LDAP objects retrieved from source LDAP")
146
147 update_objects = []
148
149 for r in searchresult:
150 dn = self.__adapt_dn(r[0])
151 d = ldap.cidict.cidict(r[1])
152
153 if not self.__is_dn_included(dn):
154 self.notify_excluded(dn)
155 else:
156 objectclasses = d["objectclass"]
157
158 newObjectclasses = []
159 for o in objectclasses:
160 if o.lower() in self.classmap:
161 new_oc = self.classmap[o.lower()]
162 if new_oc not in newObjectclasses:
163 newObjectclasses.append(new_oc)
164 else:
165 if o not in newObjectclasses:
166 newObjectclasses.append(o)
167
168 d["objectclass"] = newObjectclasses
169
170 for a in d.keys():
171 attr = a
172 if self.attrmap.has_key(a.lower()):
173 attr = self.attrmap[attr].lower()
174 if attr.lower() != a.lower():
175 values = d[a]
176 del d[a]
177 d[attr] = values
178
179 update_objects.append((dn, d))
180 return update_objects
181
182
183 def _get_dest_entry(self, dn, entry):
184 """
185 In the destination LDAP, the objects should be named
186 according to options.renameattr.
187 """
188 attrlist = self.options.attrlist
189
190 existingDestDn = None
191 existingDestEntry = None
192 if self.options.renameattr and entry.has_key(self.options.renameattr):
193 searchresult = self.con.search_s(
194 self.destbasedn,
195 ldap.SCOPE_SUBTREE,
196 '%s=%s' % (self.options.renameattr, entry[self.options.renameattr][0]), attrlist)
197 if searchresult is not None and len(searchresult) > 0:
198 existingDestDn, existingDestEntry = searchresult[0]
199 if existingDestDn.lower() != dn.lower():
200 self.con.modrdn_s(existingDestDn, dn)
201 self.notify_renamed(existingDestDn, dn,
202 existingDestEntry[self.options.renameattr][0],
203 entry[self.options.renameattr][0],
204 options)
205 if existingDestDn is None:
206 searchresult = self.con.search_s(dn, ldap.SCOPE_BASE, 'objectclass=*', attrlist)
207 existingDestDn, existingDestEntry = searchresult[0]
208 return (existingDestDn, existingDestEntry)
209
210
211 def __handle_pwdAccountLockedTime(self, dn, entry, now, max_age):
212 # hack for syncing accounts locked by password policy
213 do_unlock = False
214 if self.options.pwd_max_days > 0 and entry.has_key('pwdChangedTime'):
215 # print "pwdChangedTime set for",dn
216 pwdChange = entry['pwdChangedTime'][0]
217 d = dateutil.parser.parse(pwdChange)
218 if (now-d) > max_age:
219 entry['pwdAccountLockedTime'] = ['000001010000Z']
220 self.logger.info("locking {} {}".format(dn, pwdChange))
221 else:
222 # pwdAccountLockedTime is a operational attribute,
223 # and therefore not part of entry.
224 # Do extra search to retrieve attribute.
225 searchresult = self.con.search_s(
226 dn, ldap.SCOPE_BASE,
227 "objectclass=*", attrlist=['pwdAccountLockedTime'])
228 tmp_dn, tmp_entry = searchresult[0]
229 if tmp_entry.has_key('pwdAccountLockedTime'):
230 do_unlock = True
231 return do_unlock
232
233
234 def _syncLdapObject(self, srcDn, srcAttributes):
235 tzutc = dateutil.tz.gettz('UTC')
236 now = datetime.datetime.now(tzutc)
237 max_age = datetime.timedelta(days=self.options.pwd_max_days)
238
239 objectClasses = srcAttributes['objectClass']
240 srcAttributes['objectClass'] = [oc for oc in objectClasses if oc.lower() not in self.junk_objectclasses]
241
242 try:
243 destDn, destAttributes = self._get_dest_entry(srcDn, srcAttributes)
244
245 # hack for syncing accounts locked by password policy
246 do_unlock = self.__handle_pwdAccountLockedTime(srcDn, srcAttributes, now, max_age)
247
248 mod_attrs = ldap.modlist.modifyModlist(destAttributes, srcAttributes)
249
250 # hack for unlocking, see above
251 if do_unlock:
252 self.logger.info("unlocking {} {}".format(destDn, 'pwdAccountLockedTime'))
253 mod_attrs.append((ldap.MOD_DELETE, 'pwdAccountLockedTime', None))
254
255 if self.options.attrlist is not None:
256 mod_attrs = [a for a in mod_attrs if a[1].lower() in self.options.attrlist]
257
258 if self.junk_attrs is not None:
259 mod_attrs = [a for a in mod_attrs if a[1].lower() not in self.junk_attrs]
260
261 if mod_attrs:
262 try:
263 self.logger.debug('mod_attrs: ' + str(mod_attrs))
264 self.con.modify_s(srcDn, mod_attrs)
265 self.notify_modified(srcDn)
266 except:
267 self.logger.exception('modify failed')
268 self.notify_modified(srcDn, False)
269 else:
270 self.notify_unchanged(srcDn)
271
272 except ldap.NO_SUCH_OBJECT:
273 if not self.options.updateonly:
274 try:
275 entry = ldap.modlist.addModlist(srcAttributes, self.junk_attrs)
276 self.con.add_s(srcDn, entry)
277 self.notify_created(srcDn)
278 except (ldap.OBJECT_CLASS_VIOLATION,
279 ldap.NO_SUCH_OBJECT,
280 ldap.CONSTRAINT_VIOLATION) as e:
281 #print(e)
282 self.notify_created(srcDn, False)
283
284
285 def __syncLdapDestination(self, update_objects):
286 logger.debug("writing data to destination LDAP")
287 for obj in update_objects:
288 dn, entry = obj
289 self._syncLdapObject(dn, entry)
290
291
292 def __deleteDestLdapObjects(self, update_objects):
293 """
294 Remove all LDAP objects in destination LDAP server
295 that did not come from the source LDAP objects
296 and are not excluded.
297 """
298
299 searchresult = self.con.search_s(self.destbasedn, ldap.SCOPE_SUBTREE, self.options.filter)
300 existing = [x[0].lower() for x in searchresult]
301
302 morituri = existing
303
304 if self.destbasedn.lower() in existing:
305 morituri.remove(self.destbasedn.lower())
306
307 for obj in update_objects:
308 dn, entry = obj
309 if dn.lower() in existing:
310 morituri.remove(dn.lower())
311 for dn in morituri:
312 if self.__is_dn_included(dn):
313 try:
314 self.con.delete_s(dn)
315 self.notify_deleted(dn)
316 except:
317 self.notify_deleted(dn, False)
318
319
320 def sync(self, searchresult):
321 """
322 Synchronize entries from searchresult to destination LDAP server.
323 """
324 if len(searchresult) == 0:
325 self.logger.error("empty source, aborting")
326 return
327
328 self._dest_ldap_connect()
329
330 update_objects = self.__adapt_source_ldap_objects(searchresult)
331 self.__syncLdapDestination(update_objects)
332 if self.options.delete and not self.options.updateonly:
333 self.__deleteDestLdapObjects(update_objects)
334 self.con.unbind()
335
336 self.__log_summary(True)
337
338
339 def __log_summary(self, show_failed=True, show_ok=False):
340 result = self.result
341 for action in result.keys():
342 ok = len(result[action]['ok'])
343 failed = len(result[action]['failed'])
344 print("{} (ok: {}, failed: {})".format(action, ok, failed))
345
346 if ok > 0 and (show_ok or ok <= 3):
347 print("succeeded:")
348 print("\n".join(result[action]['ok']))
349
350 if failed > 0 and (show_failed or failed <= 3):
351 print("failed:")
352 print("\n".join(result[action]['failed']))
353 print()
354
355 def get_short_dn(self, dn):
356 return dn.lower().replace(',' + self.srcbasedn.lower(), '')
357
358 def notify_unchanged(self, dn):
359 #logger.debug(u'{} unchanged'.format(self.get_short_dn(dn)))
360 self.result['unmodified']['ok'].append(dn)
361
362 def notify_excluded(self, dn):
363 #logger.debug(u'{} unchanged'.format(self.get_short_dn(dn)))
364 self.result['excluded']['ok'].append(dn)
365
366 def notify_created(self, dn, ok=True):
367 if ok:
368 logger.debug(u'{} created'.format(self.get_short_dn(dn)))
369 self.result['add']['ok'].append(dn)
370 else:
371 self.logger.warning(u"failed to add {}".format(dn))
372 self.result['add']['failed'].append(dn)
373
374 def notify_modified(self, dn, ok=True):
375 if ok:
376 logger.debug(u'{} modified'.format(self.get_short_dn(dn)))
377 self.result['update']['ok'].append(dn)
378 else:
379 self.logger.error(u"failed to modify {}".format(dn))
380 self.result['update']['failed'].append(dn)
381
382 def notify_deleted(self, dn, ok=True):
383 if ok:
384 logger.debug(u'{} deleted'.format(self.get_short_dn(dn)))
385 self.result['delete']['ok'].append(dn)
386 else:
387 self.logger.error(u"failed to delete {}".format(dn))
388 self.result['delete']['failed'].append(dn)
389
390 def notify_renamed(self, dn, newdn, uid, newuid, options):
391 print(u"renamed {} -> {}".format(dn, newdn))
392 subprocess.check_call(
393 "%s %s %s %s %s" % (options.renamecommand, dn, newdn, uid, newuid),
394 shell=True)
395
396
397
398class SyncReplConsumer(ReconnectLDAPObject, SyncreplConsumer):
399 """
400 Syncrepl Consumer interface
401 """
402
403 def __init__(self, dest, syncrepl_entry_callback, *args, **kwargs):
404 self.logger = logging.getLogger()
405 # Initialise the LDAP Connection first
406 ldap.ldapobject.ReconnectLDAPObject.__init__(self, *args, **kwargs)
407 # We need this for later internal use
408 self.__presentUUIDs = dict()
409 self.cookie = None
410 self.dest_ldap = dest
411 self.syncrepl_entry_callback = syncrepl_entry_callback
412
413 def syncrepl_get_cookie(self):
414 return self.cookie
415
416 def syncrepl_set_cookie(self, cookie):
417 self.cookie = cookie
418
419 def syncrepl_entry(self, dn, attributes, uuid):
420 # First we determine the type of change we have here
421 # (and store away the previous data for later if needed)
422 if uuid in self.__presentUUIDs:
423 change_type = 'modify'
424 else:
425 change_type = 'add'
426 # Now we store our knowledge of the existence of this entry
427 self.__presentUUIDs[uuid] = dn
428 # Debugging
429 logger.debug('{}: {} ({})'.format(dn, change_type, ",".join(attributes.keys())))
430 # If we have a cookie then this is not our first time being run,
431 # so it must be a change
432 if self.cookie is not None:
433 self.syncrepl_entry_callback(dn, attributes)
434
435
436 def syncrepl_delete(self, uuids):
437 """ syncrepl_delete """
438 # Make sure we know about the UUID being deleted, just in case...
439 uuids = [uuid for uuid in uuids if uuid in self.__presentUUIDs]
440 # Delete all the UUID values we know of
441 for uuid in uuids:
442 logger.debug('detected deletion of entry {} ({})', uuid, self.__presentUUIDs[uuid])
443 del self.__presentUUIDs[uuid]
444
445 def syncrepl_present(self, uuids, refreshDeletes=False):
446 """ called on initial sync """
447 if uuids is not None:
448 self.logger.debug('uuids: {}'.format(','.join(uuids)))
449 # If we have not been given any UUID values,
450 # then we have recieved all the present controls...
451 if uuids is None:
452 # We only do things if refreshDeletes is false as the syncrepl
453 # extension will call syncrepl_delete instead when it detects a
454 # delete notice
455 if not refreshDeletes:
456 deletedEntries = [
457 uuid for uuid in self.__presentUUIDs
458 ]
459 self.syncrepl_delete(deletedEntries)
460 # Phase is now completed, reset the list
461 self.__presentUUIDs = {}
462 else:
463 # Note down all the UUIDs we have been sent
464 for uuid in uuids:
465 self.__presentUUIDs[uuid] = True
466
467
468 def syncrepl_refreshdone(self):
469 self.logger.info('Initial synchronization is now done, persist phase begins')
470 #self.logger.debug('UUIDs:\n' + '\n'.join(self.__presentUUIDs))
471
472
473
474class LdapSyncRepl(LdapSync):
475 def __init__(self, destsrv,
476 destadmindn, destadminpw,
477 basedn, destbasedn,
478 options=Options(), source_ldap_url_obj=None):
479 # Install our signal handlers
480 signal.signal(signal.SIGTERM, self.shutdown)
481 self.watcher_running = False
482 self.source_ldap_url_obj = source_ldap_url_obj
483 self.ldap_credentials = False
484 self.source_ldap_connection = None
485 super(LdapSyncRepl, self).__init__(destsrv,
486 destadmindn, destadminpw,
487 basedn, destbasedn, options)
488
489
490 def sync(self):
491 self._dest_ldap_connect()
492 self.watcher_running = True
493 while self.watcher_running:
494 self.logger.info('Connecting to source LDAP server')
495 # Prepare the LDAP server connection (triggers the connection as well)
496 self.source_ldap_connection = SyncReplConsumer(self.con,
497 self.perform_application_sync_callback,
498 self.source_ldap_url_obj.initializeUrl())
499
500 if self.source_ldap_url_obj.who and self.source_ldap_url_obj.cred:
501 self.ldap_credentials = True
502 # Now we login to the LDAP server
503 try:
504 self.source_ldap_connection.simple_bind_s(
505 self.source_ldap_url_obj.who, self.source_ldap_url_obj.cred)
506 except ldap.INVALID_CREDENTIALS as e:
507 print('Login to LDAP server failed: ', str(e))
508 sys.exit(1)
509 except ldap.SERVER_DOWN:
510 print('LDAP server is down, going to retry.')
511 time.sleep(5)
512 continue
513
514 # Commence the syncing
515 self.logger.info('Staring sync process')
516 ldap_search = self.source_ldap_connection.syncrepl_search(
517 self.source_ldap_url_obj.dn or '',
518 self.source_ldap_url_obj.scope or ldap.SCOPE_SUBTREE,
519 mode='refreshAndPersist',
520 attrlist=self.source_ldap_url_obj.attrs,
521 filterstr=self.source_ldap_url_obj.filterstr or '(objectClass=*)'
522 )
523
524 try:
525 while self.source_ldap_connection.syncrepl_poll(all=1, msgid=ldap_search):
526 print(".", end="")
527 except KeyboardInterrupt:
528 # User asked to exit
529 print("aborted\n")
530 self.shutdown(None, None)
531 except Exception as e:
532 # Handle any exception
533 if self.watcher_running:
534 self.logger.exception('Encountered a problem, going to retry.')
535 time.sleep(5)
536
537 def perform_application_sync_callback(self, dn, attributes):
538 logger.debug('{}: src: {}'.format(dn, str(attributes)))
539 try:
540 self._syncLdapObject(dn, attributes)
541 except ldap.NO_SUCH_OBJECT:
542 self.logger.info("SKIPPED: {} object does not exist on target".format(dn))
543 return False
544 return True
545
546 def shutdown(self, signum, stack):
547 # Declare the needed global variables
548 self.logger.info('Shutting down!')
549
550 # We are no longer running
551 self.watcher_running = False
552
553def get_ldap_url_obj(self, configsection):
554 baseurl = 'ldap://{server}:389/{basedn}'.format(server=configsection.get('server'), basedn=configsection.get('basedn'))
555 attrs = None
556 if configsection.get('attributes') is not None:
557 attrs = configsection.get('attributes').split(',')
558 return LDAPUrl(
559 baseurl,
560 dn=configsection.get('baseDn'),
561 who=configsection.get('bindDn'),
562 cred=configsection.get('basePassword'),
563 filterstr=configsection.get('filter'),
564 attrs=attrs
565 )
566
567
568if __name__ == "__main__":
569 logging.basicConfig(format='%(levelname)s %(module)s.%(funcName)s: %(message)s', level=logging.INFO)
570 logger = logging.getLogger()
571
572 args = getArguments()
573 if args.debug:
574 logger.setLevel(logging.DEBUG)
575 conffile = args.configfile
576
577 config = ConfigParser()
578 config.read(conffile)
579
580 srcfile = None
581 try:
582 srcfile = config.get("source", "file")
583 except:
584 pass
585
586 basedn = config.get("source", "baseDn")
587 filterstr = config.get("source", "filter", fallback=None)
588
589 if srcfile is None:
590 srv = config.get("source", "server")
591 admindn = config.get("source", "bindDn")
592 adminpw = config.get("source", "bindPassword")
593 starttls = config.getboolean("source", "starttls")
594
595 destsrv = config.get("destination", "server")
596 destadmindn = config.get("destination", "bindDn")
597 destadminpw = config.get("destination", "bindPassword")
598 destbasedn = config.get("destination", "baseDn")
599 try:
600 rdn = config.get("destination", "rdn")
601 logger.warning("setting rdn is currently ignored")
602 except:
603 pass
604
605 options = Options()
606 try:
607 options.exclude = config.get("destination", "excludesubtree").lower()
608 except:
609 pass
610
611 options.updateonly = not config.getboolean("destination", "create", fallback=False)
612 options.delete = config.getboolean("destination", "delete", fallback=False)
613 options.starttls = config.getboolean("destination", "starttls", fallback=False)
614 options.renameattr = config.get("destination", "detectRename", fallback=None)
615 options.renamecommand = config.get("destination", "detectRename", fallback=None)
616 options.pwd_max_days = int(config.get("source", "pwd_max_days", fallback=0))
617 options.filter = filterstr
618
619 # Set source.attrlist as global option.
620 # If source would use less attributes than dest,
621 # all attributes not retrieved from source would be deleted from dest
622 try:
623 options.attrlist = config.get("source", "attributes").split(",")
624 except:
625 options.attrlist = None
626
627 if config.get('source', 'mode', fallback=None) == 'syncrepl':
628 ldapsync = LdapSyncRepl(
629 destsrv, destadmindn, destadminpw, basedn, destbasedn,
630 options,
631 source_ldap_url_obj=get_ldap_url_obj(config['source']))
632 ldapsync.sync()
633 else:
634 if srcfile:
635 objects = readLDIFSource(srcfile)
636 else:
637 objects = readLdapSource(srv, admindn, adminpw,
638 basedn, filterstr, options.attrlist, starttls)
639
640 ldapsync = LdapSync(destsrv, destadmindn, destadminpw, basedn, destbasedn, options)
641 ldapsync.sync(objects)
Note: See TracBrowser for help on using the repository browser.