source4/torture/drs/python/link_conflicts.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Tests replication scenarios that involve conflicting linked attribute
   5 # information between the 2 DCs.
   6 #
   7 # Copyright (C) Catalyst.Net Ltd. 2017
   8 #
   9 # This program is free software; you can redistribute it and/or modify
  10 # it under the terms of the GNU General Public License as published by
  11 # the Free Software Foundation; either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21 #
  22
  23 #
  24 # Usage:
  25 #  export DC1=dc1_dns_name
  26 #  export DC2=dc2_dns_name
  27 #  export SUBUNITRUN=$samba4srcdir/scripting/bin/subunitrun
  28 #  PYTHONPATH="$PYTHONPATH:$samba4srcdir/torture/drs/python" $SUBUNITRUN link_conflicts -U"$DOMAIN/$DC_USERNAME"%"$DC_PASSWORD"
  29 #
  30
  31 import drs_base
  32 import samba.tests
  33 import ldb
  34 from ldb import SCOPE_BASE
  35 import random
  36 import time
  37
  38 from drs_base import AbstractLink
  39 from samba.dcerpc import drsuapi, misc
  40
  41 # specifies the order to sync DCs in
  42 DC1_TO_DC2 = 1
  43 DC2_TO_DC1 = 2
  44
  45 class DrsReplicaLinkConflictTestCase(drs_base.DrsBaseTestCase):
  46     def setUp(self):
  47         super(DrsReplicaLinkConflictTestCase, self).setUp()
  48
  49         # add some randomness to the test OU. (Deletion of the last test's
  50         # objects can be slow to replicate out. So the OU created by a previous
  51         # testenv may still exist at this point).
  52         rand = random.randint(1, 10000000)
  53         self.base_dn = self.ldb_dc1.get_default_basedn()
  54         self.ou = "OU=test_link_conflict%d,%s" %(rand, self.base_dn)
  55         self.ldb_dc1.add({
  56             "dn": self.ou,
  57             "objectclass": "organizationalUnit"})
  58
  59         (self.drs, self.drs_handle) = self._ds_bind(self.dnsname_dc1)
  60         (self.drs2, self.drs2_handle) = self._ds_bind(self.dnsname_dc2)
  61
  62         # disable replication for the tests so we can control at what point
  63         # the DCs try to replicate
  64         self._disable_inbound_repl(self.dnsname_dc1)
  65         self._disable_inbound_repl(self.dnsname_dc2)
  66
  67     def tearDown(self):
  68         # re-enable replication
  69         self._enable_inbound_repl(self.dnsname_dc1)
  70         self._enable_inbound_repl(self.dnsname_dc2)
  71         self.ldb_dc1.delete(self.ou, ["tree_delete:1"])
  72         super(DrsReplicaLinkConflictTestCase, self).tearDown()
  73
  74     def get_guid(self, samdb, dn):
  75         """Returns an object's GUID (in string format)"""
  76         res = samdb.search(base=dn, attrs=["objectGUID"], scope=ldb.SCOPE_BASE)
  77         return self._GUID_string(res[0]['objectGUID'][0])
  78
  79     def add_object(self, samdb, dn, objectclass="organizationalunit"):
  80         """Adds an object"""
  81         samdb.add({"dn": dn, "objectclass": objectclass})
  82         return self.get_guid(samdb, dn)
  83
  84     def modify_object(self, samdb, dn, attr, value):
  85         """Modifies an attribute for an object"""
  86         m = ldb.Message()
  87         m.dn = ldb.Dn(samdb, dn)
  88         m[attr] = ldb.MessageElement(value, ldb.FLAG_MOD_ADD, attr)
  89         samdb.modify(m)
  90
  91     def add_link_attr(self, samdb, source_dn, attr, target_dn):
  92         """Adds a linked attribute between 2 objects"""
  93         # add the specified attribute to the source object
  94         self.modify_object(samdb, source_dn, attr, target_dn)
  95
  96     def del_link_attr(self, samdb, src, attr, target):
  97         m = ldb.Message()
  98         m.dn = ldb.Dn(samdb, src)
  99         m[attr] = ldb.MessageElement(target, ldb.FLAG_MOD_DELETE, attr)
 100         samdb.modify(m)
 101
 102     def sync_DCs(self, sync_order=DC1_TO_DC2):
 103         """Manually syncs the 2 DCs to ensure they're in sync"""
 104         if sync_order == DC1_TO_DC2:
 105             # sync DC1-->DC2, then DC2-->DC1
 106             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1)
 107             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2)
 108         else:
 109             # sync DC2-->DC1, then DC1-->DC2
 110             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2)
 111             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1)
 112
 113     def ensure_unique_timestamp(self):
 114         """Waits a second to ensure a unique timestamp between 2 objects"""
 115         time.sleep(1)
 116
 117     def unique_dn(self, obj_name):
 118         """Returns a unique object DN"""
 119         # Because we run each test case twice, we need to create a unique DN so
 120         # that the 2nd run doesn't hit objects that already exist. Add some
 121         # randomness to the object DN to make it unique
 122         rand = random.randint(1, 10000000)
 123         return "%s-%d,%s" %(obj_name, rand, self.ou)
 124
 125     def assert_attrs_match(self, res1, res2, attr, expected_count):
 126         """
 127         Asserts that the search results contain the expected number of
 128         attributes and the results match on both DCs
 129         """
 130         actual_len = len(res1[0][attr])
 131         self.assertTrue(actual_len == expected_count,
 132                         "Expected %u %s attributes, but got %u" %(expected_count,
 133                                                                   attr, actual_len))
 134         actual_len = len(res2[0][attr])
 135         self.assertTrue(actual_len == expected_count,
 136                         "Expected %u %s attributes, but got %u" %(expected_count,
 137                                                                   attr, actual_len))
 138
 139         # check DCs both agree on the same linked attributes
 140         for val in res1[0][attr]:
 141             self.assertTrue(val in res2[0][attr],
 142                             "%s '%s' not found on DC2" %(attr, val))
 143
 144     def _check_replicated_links(self, src_obj_dn, expected_links):
 145         """Checks that replication sends back the expected linked attributes"""
 146
 147         hwm = drsuapi.DsReplicaHighWaterMark()
 148         hwm.tmp_highest_usn = 0
 149         hwm.reserved_usn = 0
 150         hwm.highest_usn = 0
 151
 152         self._check_replication([src_obj_dn],
 153                                 drsuapi.DRSUAPI_DRS_WRIT_REP,
 154                                 dest_dsa=None,
 155                                 drs_error=drsuapi.DRSUAPI_EXOP_ERR_SUCCESS,
 156                                 nc_dn_str=src_obj_dn,
 157                                 exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 158                                 expected_links=expected_links,
 159                                 highwatermark=hwm)
 160
 161         # Check DC2 as well
 162         self.set_test_ldb_dc(self.ldb_dc2)
 163
 164         self._check_replication([src_obj_dn],
 165                                 drsuapi.DRSUAPI_DRS_WRIT_REP,
 166                                 dest_dsa=None,
 167                                 drs_error=drsuapi.DRSUAPI_EXOP_ERR_SUCCESS,
 168                                 nc_dn_str=src_obj_dn,
 169                                 exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 170                                 expected_links=expected_links,
 171                                 highwatermark=hwm,
 172                                 drs=self.drs2, drs_handle=self.drs2_handle)
 173         self.set_test_ldb_dc(self.ldb_dc1)
 174
 175     def _test_conflict_single_valued_link(self, sync_order):
 176         """
 177         Tests a simple single-value link conflict, i.e. each DC adds a link to
 178         the same source object but linking to different targets.
 179         """
 180         src_ou = self.unique_dn("OU=src")
 181         src_guid = self.add_object(self.ldb_dc1, src_ou)
 182         self.sync_DCs()
 183
 184         # create a unique target on each DC
 185         target1_ou = self.unique_dn("OU=target1")
 186         target2_ou = self.unique_dn("OU=target2")
 187
 188         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 189         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 190
 191         # link the test OU to the respective targets created
 192         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 193         self.ensure_unique_timestamp()
 194         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 195
 196         # sync the 2 DCs
 197         self.sync_DCs(sync_order=sync_order)
 198
 199         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 200                                   scope=SCOPE_BASE, attrs=["managedBy"])
 201         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 202                                   scope=SCOPE_BASE, attrs=["managedBy"])
 203
 204         # check the object has only have one occurence of the single-valued
 205         # attribute and it matches on both DCs
 206         self.assert_attrs_match(res1, res2, "managedBy", 1)
 207
 208         self.assertTrue(res1[0]["managedBy"][0] == target2_ou,
 209                         "Expected most recent update to win conflict")
 210
 211         # we can't query the deleted links over LDAP, but we can check DRS
 212         # to make sure the DC kept a copy of the conflicting link
 213         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 214                              misc.GUID(src_guid), misc.GUID(target1_guid))
 215         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 216                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 217                              misc.GUID(src_guid), misc.GUID(target2_guid))
 218         self._check_replicated_links(src_ou, [link1, link2])
 219
 220
 221     def test_conflict_single_valued_link(self):
 222         # repeat the test twice, to give each DC a chance to resolve the conflict
 223         self._test_conflict_single_valued_link(sync_order=DC1_TO_DC2)
 224         self._test_conflict_single_valued_link(sync_order=DC2_TO_DC1)
 225
 226     def _test_duplicate_single_valued_link(self, sync_order):
 227         """
 228         Adds the same single-valued link on 2 DCs and checks we don't end up
 229         with 2 copies of the link.
 230         """
 231         # create unique objects for the link
 232         target_ou = self.unique_dn("OU=target")
 233         target_guid = self.add_object(self.ldb_dc1, target_ou)
 234         src_ou = self.unique_dn("OU=src")
 235         src_guid = self.add_object(self.ldb_dc1, src_ou)
 236         self.sync_DCs()
 237
 238         # link the same test OU to the same target on both DCs
 239         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target_ou)
 240         self.ensure_unique_timestamp()
 241         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target_ou)
 242
 243         # sync the 2 DCs
 244         self.sync_DCs(sync_order=sync_order)
 245
 246         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 247                                   scope=SCOPE_BASE, attrs=["managedBy"])
 248         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 249                                   scope=SCOPE_BASE, attrs=["managedBy"])
 250
 251         # check the object has only have one occurence of the single-valued
 252         # attribute and it matches on both DCs
 253         self.assert_attrs_match(res1, res2, "managedBy", 1)
 254
 255     def test_duplicate_single_valued_link(self):
 256         # repeat the test twice, to give each DC a chance to resolve the conflict
 257         self._test_duplicate_single_valued_link(sync_order=DC1_TO_DC2)
 258         self._test_duplicate_single_valued_link(sync_order=DC2_TO_DC1)
 259
 260     def _test_conflict_multi_valued_link(self, sync_order):
 261         """
 262         Tests a simple multi-valued link conflict. This adds 2 objects with the
 263         same username on 2 different DCs and checks their group membership is
 264         preserved after the conflict is resolved.
 265         """
 266
 267         # create a common link source
 268         src_dn = self.unique_dn("CN=src")
 269         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 270         self.sync_DCs()
 271
 272         # create the same user (link target) on each DC.
 273         # Note that the GUIDs will differ between the DCs
 274         target_dn = self.unique_dn("CN=target")
 275         target1_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 276         self.ensure_unique_timestamp()
 277         target2_guid = self.add_object(self.ldb_dc2, target_dn, objectclass="user")
 278
 279         # link the src group to the respective target created
 280         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 281         self.ensure_unique_timestamp()
 282         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 283
 284         # sync the 2 DCs. We expect the more recent target2 object to win
 285         self.sync_DCs(sync_order=sync_order)
 286
 287         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 288                                    scope=SCOPE_BASE, attrs=["member"])
 289         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 290                                    scope=SCOPE_BASE, attrs=["member"])
 291         target1_conflict = False
 292
 293         # we expect exactly 2 members in our test group (both DCs should agree)
 294         self.assert_attrs_match(res1, res2, "member", 2)
 295
 296         for val in res1[0]["member"]:
 297             # check the expected conflicting object was renamed
 298             self.assertFalse("CNF:%s" % target2_guid in val)
 299             if "CNF:%s" % target1_guid in val:
 300                 target1_conflict = True
 301
 302         self.assertTrue(target1_conflict,
 303                         "Expected link to conflicting target object not found")
 304
 305     def test_conflict_multi_valued_link(self):
 306         # repeat the test twice, to give each DC a chance to resolve the conflict
 307         self._test_conflict_multi_valued_link(sync_order=DC1_TO_DC2)
 308         self._test_conflict_multi_valued_link(sync_order=DC2_TO_DC1)
 309
 310     def _test_duplicate_multi_valued_link(self, sync_order):
 311         """
 312         Adds the same multivalued link on 2 DCs and checks we don't end up
 313         with 2 copies of the link.
 314         """
 315
 316         # create the link source/target objects
 317         src_dn = self.unique_dn("CN=src")
 318         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 319         target_dn = self.unique_dn("CN=target")
 320         target_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 321         self.sync_DCs()
 322
 323         # link the src group to the same target user separately on each DC
 324         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 325         self.ensure_unique_timestamp()
 326         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 327
 328         self.sync_DCs(sync_order=sync_order)
 329
 330         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 331                                    scope=SCOPE_BASE, attrs=["member"])
 332         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 333                                    scope=SCOPE_BASE, attrs=["member"])
 334
 335         # we expect to still have only 1 member in our test group
 336         self.assert_attrs_match(res1, res2, "member", 1)
 337
 338     def test_duplicate_multi_valued_link(self):
 339         # repeat the test twice, to give each DC a chance to resolve the conflict
 340         self._test_duplicate_multi_valued_link(sync_order=DC1_TO_DC2)
 341         self._test_duplicate_multi_valued_link(sync_order=DC2_TO_DC1)
 342
 343     def _test_conflict_backlinks(self, sync_order):
 344         """
 345         Tests that resolving a source object conflict fixes up any backlinks,
 346         e.g. the same user is added to a conflicting group.
 347         """
 348
 349         # create a common link target
 350         target_dn = self.unique_dn("CN=target")
 351         target_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 352         self.sync_DCs()
 353
 354         # create the same group (link source) on each DC.
 355         # Note that the GUIDs will differ between the DCs
 356         src_dn = self.unique_dn("CN=src")
 357         src1_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 358         self.ensure_unique_timestamp()
 359         src2_guid = self.add_object(self.ldb_dc2, src_dn, objectclass="group")
 360
 361         # link the src group to the respective target created
 362         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 363         self.ensure_unique_timestamp()
 364         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 365
 366         # sync the 2 DCs. We expect the more recent src2 object to win
 367         self.sync_DCs(sync_order=sync_order)
 368
 369         res1 = self.ldb_dc1.search(base="<GUID=%s>" % target_guid,
 370                                    scope=SCOPE_BASE, attrs=["memberOf"])
 371         res2 = self.ldb_dc2.search(base="<GUID=%s>" % target_guid,
 372                                    scope=SCOPE_BASE, attrs=["memberOf"])
 373         src1_backlink = False
 374
 375         # our test user should still be a member of 2 groups (check both DCs agree)
 376         self.assert_attrs_match(res1, res2, "memberOf", 2)
 377
 378         for val in res1[0]["memberOf"]:
 379             # check the conflicting object was renamed
 380             self.assertFalse("CNF:%s" % src2_guid in val)
 381             if "CNF:%s" % src1_guid in val:
 382                 src1_backlink = True
 383
 384         self.assertTrue(src1_backlink,
 385                         "Expected backlink to conflicting source object not found")
 386
 387     def test_conflict_backlinks(self):
 388         # repeat the test twice, to give each DC a chance to resolve the conflict
 389         self._test_conflict_backlinks(sync_order=DC1_TO_DC2)
 390         self._test_conflict_backlinks(sync_order=DC2_TO_DC1)
 391
 392     def _test_link_deletion_conflict(self, sync_order):
 393         """
 394         Checks that a deleted link conflicting with an active link is
 395         resolved correctly.
 396         """
 397
 398         # Add the link objects
 399         target_dn = self.unique_dn("CN=target")
 400         target_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 401         src_dn = self.unique_dn("CN=src")
 402         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 403         self.sync_DCs()
 404
 405         # add the same link on both DCs, and resolve any conflict
 406         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 407         self.ensure_unique_timestamp()
 408         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 409         self.sync_DCs(sync_order=sync_order)
 410
 411         # delete and re-add the link on one DC
 412         self.del_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 413         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 414
 415         # just delete it on the other DC
 416         self.ensure_unique_timestamp()
 417         self.del_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 418         # sanity-check the link is gone on this DC
 419         res1 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 420                                    scope=SCOPE_BASE, attrs=["member"])
 421         self.assertFalse("member" in res1[0], "Couldn't delete member attr")
 422
 423         # sync the 2 DCs. We expect the more older DC1 attribute to win
 424         # because it has a higher version number (even though it's older)
 425         self.sync_DCs(sync_order=sync_order)
 426
 427         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 428                                    scope=SCOPE_BASE, attrs=["member"])
 429         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 430                                    scope=SCOPE_BASE, attrs=["member"])
 431
 432         # our test user should still be a member of the group (check both DCs agree)
 433         self.assertTrue("member" in res1[0], "Expected member attribute missing")
 434         self.assert_attrs_match(res1, res2, "member", 1)
 435
 436     def test_link_deletion_conflict(self):
 437         # repeat the test twice, to give each DC a chance to resolve the conflict
 438         self._test_link_deletion_conflict(sync_order=DC1_TO_DC2)
 439         self._test_link_deletion_conflict(sync_order=DC2_TO_DC1)
 440
 441     def _test_obj_deletion_conflict(self, sync_order, del_target):
 442         """
 443         Checks that a receiving a new link for a deleted object gets
 444         resolved correctly.
 445         """
 446
 447         target_dn = self.unique_dn("CN=target")
 448         target_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 449         src_dn = self.unique_dn("CN=src")
 450         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 451
 452         self.sync_DCs()
 453
 454         # delete the object on one DC
 455         if del_target:
 456             search_guid = src_guid
 457             self.ldb_dc2.delete(target_dn)
 458         else:
 459             search_guid = target_guid
 460             self.ldb_dc2.delete(src_dn)
 461
 462         # add a link on the other DC
 463         self.ensure_unique_timestamp()
 464         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 465
 466         self.sync_DCs(sync_order=sync_order)
 467
 468         # the object deletion should trump the link addition.
 469         # Check the link no longer exists on the remaining object
 470         res1 = self.ldb_dc1.search(base="<GUID=%s>" % search_guid,
 471                                    scope=SCOPE_BASE, attrs=["member", "memberOf"])
 472         res2 = self.ldb_dc2.search(base="<GUID=%s>" % search_guid,
 473                                    scope=SCOPE_BASE, attrs=["member", "memberOf"])
 474
 475         self.assertFalse("member" in res1[0], "member attr shouldn't exist")
 476         self.assertFalse("member" in res2[0], "member attr shouldn't exist")
 477         self.assertFalse("memberOf" in res1[0], "member attr shouldn't exist")
 478         self.assertFalse("memberOf" in res2[0], "member attr shouldn't exist")
 479
 480     def test_obj_deletion_conflict(self):
 481         # repeat the test twice, to give each DC a chance to resolve the conflict
 482         self._test_obj_deletion_conflict(sync_order=DC1_TO_DC2, del_target=True)
 483         self._test_obj_deletion_conflict(sync_order=DC2_TO_DC1, del_target=True)
 484
 485         # and also try deleting the source object instead of the link target
 486         self._test_obj_deletion_conflict(sync_order=DC1_TO_DC2, del_target=False)
 487         self._test_obj_deletion_conflict(sync_order=DC2_TO_DC1, del_target=False)
 488
 489     def _test_full_sync_link_conflict(self, sync_order):
 490         """
 491         Checks that doing a full sync doesn't affect how conflicts get resolved
 492         """
 493
 494         # create the objects for the linked attribute
 495         src_dn = self.unique_dn("CN=src")
 496         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 497         target_dn = self.unique_dn("CN=target")
 498         target1_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 499         self.sync_DCs()
 500
 501         # add the same link on both DCs
 502         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 503         self.ensure_unique_timestamp()
 504         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 505
 506         # Do a couple of full syncs which should resolve the conflict
 507         # (but only for one DC)
 508         if sync_order == DC1_TO_DC2:
 509             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1, full_sync=True)
 510             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1, full_sync=True)
 511         else:
 512             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2, full_sync=True)
 513             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2, full_sync=True)
 514
 515         # delete and re-add the link on one DC
 516         self.del_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 517         self.ensure_unique_timestamp()
 518         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 519
 520         # just delete the link on the 2nd DC
 521         self.ensure_unique_timestamp()
 522         self.del_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 523
 524         # sync the 2 DCs. We expect DC1 to win based on version number
 525         self.sync_DCs(sync_order=sync_order)
 526
 527         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 528                                    scope=SCOPE_BASE, attrs=["member"])
 529         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 530                                    scope=SCOPE_BASE, attrs=["member"])
 531
 532         # check the membership still exits (and both DCs agree)
 533         self.assertTrue("member" in res1[0], "Expected member attribute missing")
 534         self.assert_attrs_match(res1, res2, "member", 1)
 535
 536     def test_full_sync_link_conflict(self):
 537         # repeat the test twice, to give each DC a chance to resolve the conflict
 538         self._test_full_sync_link_conflict(sync_order=DC1_TO_DC2)
 539         self._test_full_sync_link_conflict(sync_order=DC2_TO_DC1)
 540
 541     def _test_conflict_single_valued_link_deleted_winner(self, sync_order):
 542         """
 543         Tests a single-value link conflict where the more-up-to-date link value
 544         is deleted.
 545         """
 546         src_ou = self.unique_dn("OU=src")
 547         src_guid = self.add_object(self.ldb_dc1, src_ou)
 548         self.sync_DCs()
 549
 550         # create a unique target on each DC
 551         target1_ou = self.unique_dn("OU=target1")
 552         target2_ou = self.unique_dn("OU=target2")
 553
 554         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 555         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 556
 557         # add the links for the respective targets, and delete one of the links
 558         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 559         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 560         self.ensure_unique_timestamp()
 561         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 562
 563         # sync the 2 DCs
 564         self.sync_DCs(sync_order=sync_order)
 565
 566         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 567                                   scope=SCOPE_BASE, attrs=["managedBy"])
 568         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 569                                   scope=SCOPE_BASE, attrs=["managedBy"])
 570
 571         # Although the more up-to-date link value is deleted, this shouldn't
 572         # trump DC1's active link
 573         self.assert_attrs_match(res1, res2, "managedBy", 1)
 574
 575         self.assertTrue(res1[0]["managedBy"][0] == target2_ou,
 576                         "Expected active link win conflict")
 577
 578         # we can't query the deleted links over LDAP, but we can check that
 579         # the deleted links exist using DRS
 580         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 581                              misc.GUID(src_guid), misc.GUID(target1_guid))
 582         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 583                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 584                              misc.GUID(src_guid), misc.GUID(target2_guid))
 585         self._check_replicated_links(src_ou, [link1, link2])
 586
 587     def test_conflict_single_valued_link_deleted_winner(self):
 588         # repeat the test twice, to give each DC a chance to resolve the conflict
 589         self._test_conflict_single_valued_link_deleted_winner(sync_order=DC1_TO_DC2)
 590         self._test_conflict_single_valued_link_deleted_winner(sync_order=DC2_TO_DC1)
 591
 592     def _test_conflict_single_valued_link_deleted_loser(self, sync_order):
 593         """
 594         Tests a single-valued link conflict, where the losing link value is deleted.
 595         """
 596         src_ou = self.unique_dn("OU=src")
 597         src_guid = self.add_object(self.ldb_dc1, src_ou)
 598         self.sync_DCs()
 599
 600         # create a unique target on each DC
 601         target1_ou = self.unique_dn("OU=target1")
 602         target2_ou = self.unique_dn("OU=target2")
 603
 604         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 605         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 606
 607         # add the links - we want the link to end up deleted on DC2, but active on
 608         # DC1. DC1 has the better version and DC2 has the better timestamp - the
 609         # better version should win
 610         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 611         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 612         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 613         self.ensure_unique_timestamp()
 614         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 615         self.del_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 616
 617         self.sync_DCs(sync_order=sync_order)
 618
 619         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 620                                   scope=SCOPE_BASE, attrs=["managedBy"])
 621         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 622                                   scope=SCOPE_BASE, attrs=["managedBy"])
 623
 624         # check the object has only have one occurence of the single-valued
 625         # attribute and it matches on both DCs
 626         self.assert_attrs_match(res1, res2, "managedBy", 1)
 627
 628         self.assertTrue(res1[0]["managedBy"][0] == target1_ou,
 629                         "Expected most recent update to win conflict")
 630
 631         # we can't query the deleted links over LDAP, but we can check DRS
 632         # to make sure the DC kept a copy of the conflicting link
 633         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 634                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 635                              misc.GUID(src_guid), misc.GUID(target1_guid))
 636         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 637                              misc.GUID(src_guid), misc.GUID(target2_guid))
 638         self._check_replicated_links(src_ou, [link1, link2])
 639
 640     def test_conflict_single_valued_link_deleted_loser(self):
 641         # repeat the test twice, to give each DC a chance to resolve the conflict
 642         self._test_conflict_single_valued_link_deleted_loser(sync_order=DC1_TO_DC2)
 643         self._test_conflict_single_valued_link_deleted_loser(sync_order=DC2_TO_DC1)
 644
 645     def _test_conflict_existing_single_valued_link(self, sync_order):
 646         """
 647         Tests a single-valued link conflict, where the conflicting link value
 648         already exists (as inactive) on both DCs.
 649         """
 650         # create the link objects
 651         src_ou = self.unique_dn("OU=src")
 652         src_guid = self.add_object(self.ldb_dc1, src_ou)
 653
 654         target1_ou = self.unique_dn("OU=target1")
 655         target2_ou = self.unique_dn("OU=target2")
 656         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 657         target2_guid = self.add_object(self.ldb_dc1, target2_ou)
 658
 659         # add the links, but then delete them
 660         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 661         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 662         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target2_ou)
 663         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target2_ou)
 664         self.sync_DCs()
 665
 666         # re-add the links independently on each DC
 667         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 668         self.ensure_unique_timestamp()
 669         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 670
 671         # try to sync the 2 DCs (this currently fails)
 672         try:
 673             self.sync_DCs(sync_order=sync_order)
 674         except Exception, e:
 675             self.fail("Replication could not resolve link conflict: %s" % e)
 676
 677         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 678                                   scope=SCOPE_BASE, attrs=["managedBy"])
 679         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 680                                   scope=SCOPE_BASE, attrs=["managedBy"])
 681
 682         # check the object has only have one occurence of the single-valued
 683         # attribute and it matches on both DCs
 684         self.assert_attrs_match(res1, res2, "managedBy", 1)
 685
 686         # here we expect DC2 to win because it has the more recent link
 687         self.assertTrue(res1[0]["managedBy"][0] == target2_ou,
 688                         "Expected most recent update to win conflict")
 689
 690         # we can't query the deleted links over LDAP, but we can check DRS
 691         # to make sure the DC kept a copy of the conflicting link
 692         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 693                              misc.GUID(src_guid), misc.GUID(target1_guid))
 694         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 695                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 696                              misc.GUID(src_guid), misc.GUID(target2_guid))
 697         self._check_replicated_links(src_ou, [link1, link2])
 698
 699     def test_conflict_existing_single_valued_link(self):
 700         # repeat the test twice, to give each DC a chance to resolve the conflict
 701         self._test_conflict_existing_single_valued_link(sync_order=DC1_TO_DC2)
 702         self._test_conflict_existing_single_valued_link(sync_order=DC2_TO_DC1)
 703