source4/dsdb/tests/python/ad_dc_search_performance.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 from __future__ import print_function
   4
   5 import optparse
   6 import sys
   7 sys.path.insert(0, 'bin/python')
   8
   9 import os
  10 import samba
  11 import samba.getopt as options
  12 import random
  13 import tempfile
  14 import shutil
  15 import time
  16 import itertools
  17
  18 from samba.netcmd.main import cmd_sambatool
  19
  20 # We try to use the test infrastructure of Samba 4.3+, but if it
  21 # doesn't work, we are probably in a back-ported patch and trying to
  22 # run on 4.1 or something.
  23 #
  24 # Don't copy this horror into ordinary tests -- it is special for
  25 # performance tests that want to apply to old versions.
  26 try:
  27     from samba.tests.subunitrun import SubunitOptions, TestProgram
  28     ANCIENT_SAMBA = False
  29 except ImportError:
  30     ANCIENT_SAMBA = True
  31     samba.ensure_external_module("testtools", "testtools")
  32     samba.ensure_external_module("subunit", "subunit/python")
  33     from subunit.run import SubunitTestRunner
  34     import unittest
  35
  36 from samba.samdb import SamDB
  37 from samba.auth import system_session
  38 from ldb import Message, MessageElement, Dn, LdbError
  39 from ldb import FLAG_MOD_ADD, FLAG_MOD_REPLACE, FLAG_MOD_DELETE
  40 from ldb import SCOPE_BASE, SCOPE_SUBTREE, SCOPE_ONELEVEL
  41
  42 parser = optparse.OptionParser("ad_dc_performance.py [options] <host>")
  43 sambaopts = options.SambaOptions(parser)
  44 parser.add_option_group(sambaopts)
  45 parser.add_option_group(options.VersionOptions(parser))
  46
  47 if not ANCIENT_SAMBA:
  48     subunitopts = SubunitOptions(parser)
  49     parser.add_option_group(subunitopts)
  50
  51 # use command line creds if available
  52 credopts = options.CredentialsOptions(parser)
  53 parser.add_option_group(credopts)
  54 opts, args = parser.parse_args()
  55
  56
  57 if len(args) < 1:
  58     parser.print_usage()
  59     sys.exit(1)
  60
  61 host = args[0]
  62
  63 lp = sambaopts.get_loadparm()
  64 creds = credopts.get_credentials(lp)
  65
  66 random.seed(1)
  67
  68
  69 class PerfTestException(Exception):
  70     pass
  71
  72
  73 BATCH_SIZE = 1000
  74 N_GROUPS = 5
  75
  76
  77 class GlobalState(object):
  78     next_user_id = 0
  79     n_groups = 0
  80     next_linked_user = 0
  81     next_relinked_user = 0
  82     next_linked_user_3 = 0
  83     next_removed_link_0 = 0
  84
  85
  86 class UserTests(samba.tests.TestCase):
  87
  88     def add_if_possible(self, *args, **kwargs):
  89         """In these tests sometimes things are left in the database
  90         deliberately, so we don't worry if we fail to add them a second
  91         time."""
  92         try:
  93             self.ldb.add(*args, **kwargs)
  94         except LdbError:
  95             pass
  96
  97     def setUp(self):
  98         super(UserTests, self).setUp()
  99         self.state = GlobalState  # the class itself, not an instance
 100         self.lp = lp
 101         self.ldb = SamDB(host, credentials=creds,
 102                          session_info=system_session(lp), lp=lp)
 103         self.base_dn = self.ldb.domain_dn()
 104         self.ou = "OU=pid%s,%s" % (os.getpid(), self.base_dn)
 105         self.ou_users = "OU=users,%s" % self.ou
 106         self.ou_groups = "OU=groups,%s" % self.ou
 107         self.ou_computers = "OU=computers,%s" % self.ou
 108
 109         for dn in (self.ou, self.ou_users, self.ou_groups,
 110                    self.ou_computers):
 111             self.add_if_possible({
 112                 "dn": dn,
 113                 "objectclass": "organizationalUnit"})
 114
 115     def tearDown(self):
 116         super(UserTests, self).tearDown()
 117
 118     def test_00_00_do_nothing(self):
 119         # this gives us an idea of the overhead
 120         pass
 121
 122     def _prepare_n_groups(self, n):
 123         self.state.n_groups = n
 124         for i in range(n):
 125             self.add_if_possible({
 126                 "dn": "cn=g%d,%s" % (i, self.ou_groups),
 127                 "objectclass": "group"})
 128
 129     def _add_users(self, start, end):
 130         for i in range(start, end):
 131             self.ldb.add({
 132                 "dn": "cn=u%d,%s" % (i, self.ou_users),
 133                 "objectclass": "user"})
 134
 135     def _add_users_ldif(self, start, end):
 136         lines = []
 137         for i in range(start, end):
 138             lines.append("dn: cn=u%d,%s" % (i, self.ou_users))
 139             lines.append("objectclass: user")
 140             lines.append("")
 141         self.ldb.add_ldif('\n'.join(lines))
 142
 143     def _test_unindexed_search(self):
 144         expressions = [
 145             ('(&(objectclass=user)(description='
 146              'Built-in account for adminstering the computer/domain))'),
 147             '(description=Built-in account for adminstering the computer/domain)',
 148             '(objectCategory=*)',
 149             '(samaccountname=Administrator*)'
 150         ]
 151         for expression in expressions:
 152             t = time.time()
 153             for i in range(50):
 154                 self.ldb.search(self.ou,
 155                                 expression=expression,
 156                                 scope=SCOPE_SUBTREE,
 157                                 attrs=['cn'])
 158             print('%d %s took %s' % (i, expression,
 159                                      time.time() - t),
 160                   file=sys.stderr)
 161
 162     def _test_indexed_search(self):
 163         expressions = ['(objectclass=group)',
 164                        '(samaccountname=Administrator)'
 165                        ]
 166         for expression in expressions:
 167             t = time.time()
 168             for i in range(10000):
 169                 self.ldb.search(self.ou,
 170                                 expression=expression,
 171                                 scope=SCOPE_SUBTREE,
 172                                 attrs=['cn'])
 173             print('%d runs %s took %s' % (i, expression,
 174                                           time.time() - t),
 175                   file=sys.stderr)
 176
 177     def _test_complex_search(self):
 178         classes = ['samaccountname', 'objectCategory', 'dn', 'member']
 179         values = ['*', '*t*', 'g*', 'user']
 180         comparators = ['=', '<=', '>=']  # '~=' causes error
 181         maybe_not = ['!(', '']
 182         joiners = ['&', '|']
 183
 184         # The number of permuations is 18432, which is not huge but
 185         # would take hours to search. So we take a sample.
 186         all_permutations = list(itertools.product(joiners,
 187                                                   classes, classes,
 188                                                   values, values,
 189                                                   comparators, comparators,
 190                                                   maybe_not, maybe_not))
 191         random.seed(1)
 192
 193
 194
 195         for (j, c1, c2, v1, v2,
 196              o1, o2, n1, n2) in random.sample(all_permutations, 100):
 197             expression = ''.join(['(', j,
 198                                   '(', n1, c1, o1, v1,
 199                                   '))' if n1 else ')',
 200                                   '(', n2, c2, o2, v2,
 201                                   '))' if n2 else ')',
 202                                   ')'])
 203             print(expression)
 204             self.ldb.search(self.ou,
 205                             expression=expression,
 206                             scope=SCOPE_SUBTREE,
 207                             attrs=['cn'])
 208
 209     def _test_member_search(self, rounds=10):
 210         expressions = []
 211         for d in range(50):
 212             expressions.append('(member=cn=u%d,%s)' % (d + 500, self.ou_users))
 213             expressions.append('(member=u%d*)' % (d + 700,))
 214         for i in range(N_GROUPS):
 215             expressions.append('(memberOf=cn=g%d,%s)' % (i, self.ou_groups))
 216             expressions.append('(memberOf=cn=g%d*)' % (i,))
 217             expressions.append('(memberOf=cn=*%s*)' % self.ou_groups)
 218
 219         for expression in expressions:
 220             t = time.time()
 221             for i in range(rounds):
 222                 self.ldb.search(self.ou,
 223                                 expression=expression,
 224                                 scope=SCOPE_SUBTREE,
 225                                 attrs=['cn'])
 226             print('%d runs %s took %s' % (i, expression,
 227                                           time.time() - t),
 228                   file=sys.stderr)
 229
 230     def _test_add_many_users(self, n=BATCH_SIZE):
 231         s = self.state.next_user_id
 232         e = s + n
 233         self._add_users(s, e)
 234         self.state.next_user_id = e
 235
 236     def _test_add_many_users_ldif(self, n=BATCH_SIZE):
 237         s = self.state.next_user_id
 238         e = s + n
 239         self._add_users_ldif(s, e)
 240         self.state.next_user_id = e
 241
 242     def _link_user_and_group(self, u, g):
 243         m = Message()
 244         m.dn = Dn(self.ldb, "CN=g%d,%s" % (g, self.ou_groups))
 245         m["member"] = MessageElement("cn=u%d,%s" % (u, self.ou_users),
 246                                      FLAG_MOD_ADD, "member")
 247         self.ldb.modify(m)
 248
 249     def _test_link_many_users(self, n=BATCH_SIZE):
 250         self._prepare_n_groups(N_GROUPS)
 251         s = self.state.next_linked_user
 252         e = s + n
 253         for i in range(s, e):
 254             # put everyone in group 0, and one other group
 255             g = i % (N_GROUPS - 1) + 1
 256             self._link_user_and_group(i, g)
 257             self._link_user_and_group(i, 0)
 258         self.state.next_linked_user = e
 259
 260
 261     test_00_01_adding_users_1000 = _test_add_many_users
 262
 263     test_00_10_complex_search_1k_users = _test_complex_search
 264     test_00_11_unindexed_search_1k_users = _test_unindexed_search
 265     test_00_12_indexed_search_1k_users = _test_indexed_search
 266     test_00_13_member_search_1k_users = _test_member_search
 267
 268     test_01_02_adding_users_2000_ldif = _test_add_many_users_ldif
 269     test_01_03_adding_users_3000 = _test_add_many_users
 270
 271     test_01_10_complex_search_3k_users = _test_complex_search
 272     test_01_11_unindexed_search_3k_users = _test_unindexed_search
 273     test_01_12_indexed_search_3k_users = _test_indexed_search
 274
 275     def test_01_13_member_search_3k_users(self):
 276         self._test_member_search(rounds=5)
 277
 278     test_02_01_link_users_1000 = _test_link_many_users
 279     test_02_02_link_users_2000 = _test_link_many_users
 280     test_02_03_link_users_3000 = _test_link_many_users
 281
 282     test_03_10_complex_search_linked_users = _test_complex_search
 283     test_03_11_unindexed_search_linked_users = _test_unindexed_search
 284     test_03_12_indexed_search_linked_users = _test_indexed_search
 285
 286     def test_03_13_member_search_linked_users(self):
 287         self._test_member_search(rounds=2)
 288
 289 if "://" not in host:
 290     if os.path.isfile(host):
 291         host = "tdb://%s" % host
 292     else:
 293         host = "ldap://%s" % host
 294
 295
 296 if ANCIENT_SAMBA:
 297     runner = SubunitTestRunner()
 298     if not runner.run(unittest.makeSuite(UserTests)).wasSuccessful():
 299         sys.exit(1)
 300     sys.exit(0)
 301 else:
 302     TestProgram(module=__name__, opts=subunitopts)