lib ldb key_value: Add get_size method

Add the get_size method to the ldb_key_value layer, this will allow the
reindexing code to get an estimate of the number of records in the
database.

The lmdb backend returns an accurate count of the number of records in
the database withe the mdb_env_stat call.

The tdb backend does not provide a low cost method to determine the
number of records on the database.  It does provide a tdb_summary call
however this this walks the entire database.

So for tdb we use the map size divided by 500, this over estimates the counts
for small domains, but the extra memory allocated for the cache should
not be significant.

Signed-off-by: Gary Lockyer <gary@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>

diff --git a/lib/ldb/ldb_key_value/ldb_kv.h b/lib/ldb/ldb_key_value/ldb_kv.h
index 2c08b1db480ba7a468262633333e31dd60e92fa9..2eb6fdfed8bd77ec7a863db1e9ab0937729aab97 100644 (file)
--- a/lib/ldb/ldb_key_value/ldb_kv.h
+++ b/lib/ldb/ldb_key_value/ldb_kv.h
@@ -43,6 +43,7 @@ struct kv_db_ops {
        const char *(*name)(struct ldb_kv_private *ldb_kv);
        bool (*has_changed)(struct ldb_kv_private *ldb_kv);
        bool (*transaction_active)(struct ldb_kv_private *ldb_kv);
+       size_t (*get_size)(struct ldb_kv_private *ldb_kv);
 };
 
 /* this private structure is used by the key value backends in the

diff --git a/lib/ldb/ldb_mdb/ldb_mdb.c b/lib/ldb/ldb_mdb/ldb_mdb.c
index 646a67c554c21dec17f78ee87e2457368a411115..1e94d6608624d8d60b996506bd9eb05f03dfa17c 100644 (file)
--- a/lib/ldb/ldb_mdb/ldb_mdb.c
+++ b/lib/ldb/ldb_mdb/ldb_mdb.c
@@ -576,6 +576,27 @@ static bool lmdb_changed(struct ldb_kv_private *ldb_kv)
        return true;
 }
 
+/*
+ * Get the number of records in the database.
+ *
+ * The mdb_env_stat call returns an accurate count, so we return the actual
+ * number of records in the database rather than an estimate.
+ */
+static size_t lmdb_get_size(struct ldb_kv_private *ldb_kv) {
+
+       struct MDB_stat stats = {0};
+       struct lmdb_private *lmdb = ldb_kv->lmdb_private;
+       int ret = 0;
+
+       ret = mdb_env_stat(lmdb->env, &stats);
+       if (ret != 0) {
+               return 0;
+       }
+       return stats.ms_entries;
+}
+
+
+
 static struct kv_db_ops lmdb_key_value_ops = {
        .store              = lmdb_store,
        .delete             = lmdb_delete,
@@ -593,6 +614,7 @@ static struct kv_db_ops lmdb_key_value_ops = {
        .name               = lmdb_name,
        .has_changed        = lmdb_changed,
        .transaction_active = lmdb_transaction_active,
+       .get_size           = lmdb_get_size,
 };
 
 static const char *lmdb_get_path(const char *url)

diff --git a/lib/ldb/ldb_tdb/ldb_tdb.c b/lib/ldb/ldb_tdb/ldb_tdb.c
index 812ddd3e3893640351eca31be415736c02974785..9f16040659ba627a9bc8cb1ee74e8c8755c19c74 100644 (file)
--- a/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -400,6 +400,23 @@ static bool ltdb_transaction_active(struct ldb_kv_private *ldb_kv)
        return tdb_transaction_active(ldb_kv->tdb);
 }
 
+/*
+ * Get an estimate of the number of records in a tdb database.
+ *
+ * This implementation will overestimate the number of records in a sparsely
+ * populated database. The size estimate is only used for allocating
+ * an in memory tdb to cache index records during a reindex, overestimating
+ * the contents is acceptable, and preferable to underestimating
+ */
+#define RECORD_SIZE 500
+static size_t ltdb_get_size(struct ldb_kv_private *ldb_kv)
+{
+       size_t map_size = tdb_map_size(ldb_kv->tdb);
+       size_t size = map_size / RECORD_SIZE;
+
+       return size;
+}
+
 static const struct kv_db_ops key_value_ops = {
     .store = ltdb_store,
     .delete = ltdb_delete,
@@ -417,6 +434,7 @@ static const struct kv_db_ops key_value_ops = {
     .name = ltdb_name,
     .has_changed = ltdb_changed,
     .transaction_active = ltdb_transaction_active,
+    .get_size = ltdb_get_size,
 };
 
 /*

diff --git a/lib/ldb/tests/ldb_kv_ops_test.c b/lib/ldb/tests/ldb_kv_ops_test.c
index d6a4dc058e5348fa3c3d77080d334b2d2a3ce78d..9fa78f9ae5aa950a7a5533e2e149019362b45227 100644 (file)
--- a/lib/ldb/tests/ldb_kv_ops_test.c
+++ b/lib/ldb/tests/ldb_kv_ops_test.c
@@ -43,6 +43,9 @@
  * - supports iteration over all records in the database
  * - supports the update_in_iterate operation allowing entries to be
  *   re-keyed.
+ * - has a get_size implementation that returns an estimate of the number of
+ *   records in the database.  Note that this can be an estimate rather than
+ *   an accurate size.
  */
 #include <stdarg.h>
 #include <stddef.h>
@@ -1523,6 +1526,71 @@ static void test_delete_transaction_isolation(void **state)
 }
 
 
+/*
+ * Test that get_size returns a sensible estimate of the number of records
+ * in the database.
+ */
+static void test_get_size(void **state)
+{
+       int ret;
+       struct test_ctx *test_ctx = talloc_get_type_abort(*state,
+                                                         struct test_ctx);
+       struct ldb_kv_private *ldb_kv = get_ldb_kv(test_ctx->ldb);
+       uint8_t key_val[] = "TheKey";
+       struct ldb_val key = {
+               .data   = key_val,
+               .length = sizeof(key_val)
+       };
+
+       uint8_t value[] = "The record contents";
+       struct ldb_val data = {
+               .data    = value,
+               .length = sizeof(value)
+       };
+       size_t size = 0;
+
+       int flags = 0;
+       TALLOC_CTX *tmp_ctx;
+
+       tmp_ctx = talloc_new(test_ctx);
+       assert_non_null(tmp_ctx);
+
+       size = ldb_kv->kv_ops->get_size(ldb_kv);
+#if defined(TEST_LMDB)
+       assert_int_equal(2, size);
+#else
+       /*
+        * The tdb implementation of get_size over estimates for sparse files
+        * which is perfectly acceptable for it's intended use.
+        */
+       assert_true( size > 2500);
+#endif
+
+       /*
+        * Begin a transaction
+        */
+       ret = ldb_kv->kv_ops->begin_write(ldb_kv);
+       assert_int_equal(ret, 0);
+
+       /*
+        * Write the record
+        */
+       ret = ldb_kv->kv_ops->store(ldb_kv, key, data, flags);
+       assert_int_equal(ret, 0);
+
+       /*
+        * Commit the transaction
+        */
+       ret = ldb_kv->kv_ops->finish_write(ldb_kv);
+       assert_int_equal(ret, 0);
+
+       size = ldb_kv->kv_ops->get_size(ldb_kv);
+#ifdef TEST_LMDB
+       assert_int_equal(3, size);
+#endif
+       talloc_free(tmp_ctx);
+}
+
 int main(int argc, const char **argv)
 {
        const struct CMUnitTest tests[] = {
@@ -1570,6 +1638,10 @@ int main(int argc, const char **argv)
                        test_delete_transaction_isolation,
                        setup,
                        teardown),
+               cmocka_unit_test_setup_teardown(
+                       test_get_size,
+                       setup,
+                       teardown),
        };
 
        return cmocka_run_group_tests(tests, NULL, NULL);

diff --git a/lib/ldb/wscript b/lib/ldb/wscript
index 0fdd888f5336d23873d6d490ea3fb5d280c39bff..743dae18357bec3e71b9220c6bbf6561533881bd 100644 (file)
--- a/lib/ldb/wscript
+++ b/lib/ldb/wscript
@@ -526,7 +526,7 @@ def build(bld):
 
             bld.SAMBA_BINARY('ldb_mdb_kv_ops_test',
                              source='tests/ldb_kv_ops_test.c',
-                             cflags='-DTEST_BE=\"mdb\"',
+                             cflags='-DTEST_BE=\"mdb\" -DTEST_LMDB=1',
                              deps='cmocka ldb',
                              install=False)
         else: