From 2a9060641757937ad764685ec35507629ce6283e Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Wed, 4 May 2016 17:01:15 +1200 Subject: [PATCH] dsdb: Remove 120 second delay and USN from schema refresh check We now refresh it once the schema changes, so that replication can proceed right away. We use the sequence number in the metadata.tdb. The previous commit added a cache for this value, protected by tdb_seqnum(). metadata.tdb is now opened at startup to provide this support. Note that while still supported, schemaUpdateNow is essentially rudundent: instead, to ensure we increment the sequence number correctly, we unify that check into repl_meta_data at the transaction close. Signed-off-by: Andrew Bartlett Reviewed-by: Garming Sam --- source4/dsdb/repl/replicated_objects.c | 80 ++--------- source4/dsdb/samdb/ldb_modules/acl.c | 2 - .../dsdb/samdb/ldb_modules/repl_meta_data.c | 17 +++ source4/dsdb/samdb/ldb_modules/schema_load.c | 124 +++++++----------- source4/dsdb/schema/schema.h | 5 - source4/dsdb/schema/schema_init.c | 6 - 6 files changed, 76 insertions(+), 158 deletions(-) diff --git a/source4/dsdb/repl/replicated_objects.c b/source4/dsdb/repl/replicated_objects.c index 919dd46ca1f..88ba598ba6b 100644 --- a/source4/dsdb/repl/replicated_objects.c +++ b/source4/dsdb/repl/replicated_objects.c @@ -883,83 +883,25 @@ WERROR dsdb_replicated_objects_commit(struct ldb_context *ldb, * a schema cache being refreshed from database. */ if (working_schema) { - struct ldb_message *msg; - struct ldb_request *req; - - /* Force a reload */ - working_schema->last_refresh = 0; + /* Reload the schema */ new_schema = dsdb_get_schema(ldb, tmp_ctx); - /* TODO: + /* TODO: * If dsdb_get_schema() fails, we just fall back * to what we had. However, the database is probably * unable to operate for other users from this * point... */ - if (new_schema && used_global_schema) { - dsdb_make_schema_global(ldb, new_schema); - } else if (used_global_schema) { - DEBUG(0,("Failed to re-load schema after commit of transaction\n")); - dsdb_set_global_schema(ldb); - TALLOC_FREE(tmp_ctx); - return WERR_INTERNAL_ERROR; - } else { - DEBUG(0,("Failed to re-load schema after commit of transaction\n")); + if (new_schema == NULL || new_schema == working_schema) { + DEBUG(0,("Failed to re-load schema after commit of transaction (working: %p/%llu, new: %p/%llu)\n", + new_schema, (unsigned long long)new_schema->metadata_usn, + working_schema, (unsigned long long)working_schema->metadata_usn)); dsdb_reference_schema(ldb, cur_schema, false); + if (used_global_schema) { + dsdb_set_global_schema(ldb); + } TALLOC_FREE(tmp_ctx); return WERR_INTERNAL_ERROR; - } - msg = ldb_msg_new(tmp_ctx); - if (msg == NULL) { - TALLOC_FREE(tmp_ctx); - return WERR_NOMEM; - } - msg->dn = ldb_dn_new(msg, ldb, ""); - if (msg->dn == NULL) { - TALLOC_FREE(tmp_ctx); - return WERR_NOMEM; - } - - ret = ldb_msg_add_string(msg, "schemaUpdateNow", "1"); - if (ret != LDB_SUCCESS) { - TALLOC_FREE(tmp_ctx); - return WERR_INTERNAL_ERROR; - } - - ret = ldb_build_mod_req(&req, ldb, objects, - msg, - NULL, - NULL, - ldb_op_default_callback, - NULL); - - if (ret != LDB_SUCCESS) { - TALLOC_FREE(tmp_ctx); - return WERR_DS_DRA_INTERNAL_ERROR; - } - - ret = ldb_transaction_start(ldb); - if (ret != LDB_SUCCESS) { - TALLOC_FREE(tmp_ctx); - DEBUG(0, ("Autotransaction start failed\n")); - return WERR_DS_DRA_INTERNAL_ERROR; - } - - ret = ldb_request(ldb, req); - if (ret == LDB_SUCCESS) { - ret = ldb_wait(req->handle, LDB_WAIT_ALL); - } - - if (ret == LDB_SUCCESS) { - ret = ldb_transaction_commit(ldb); - } else { - DEBUG(0, ("Schema update now failed: %s\n", - ldb_errstring(ldb))); - ldb_transaction_cancel(ldb); - } - - if (ret != LDB_SUCCESS) { - DEBUG(0, ("Commit failed: %s\n", ldb_errstring(ldb))); - TALLOC_FREE(tmp_ctx); - return WERR_DS_INTERNAL_FAILURE; + } else if (used_global_schema) { + dsdb_make_schema_global(ldb, new_schema); } } diff --git a/source4/dsdb/samdb/ldb_modules/acl.c b/source4/dsdb/samdb/ldb_modules/acl.c index 2a98cd85e75..27d4e7659cc 100644 --- a/source4/dsdb/samdb/ldb_modules/acl.c +++ b/source4/dsdb/samdb/ldb_modules/acl.c @@ -1651,7 +1651,6 @@ static int acl_search_update_confidential_attrs(struct acl_context *ac, } if ((ac->schema == data->cached_schema_ptr) && - (ac->schema->loaded_usn == data->cached_schema_loaded_usn) && (ac->schema->metadata_usn == data->cached_schema_metadata_usn)) { return LDB_SUCCESS; @@ -1687,7 +1686,6 @@ static int acl_search_update_confidential_attrs(struct acl_context *ac, } data->cached_schema_ptr = ac->schema; - data->cached_schema_loaded_usn = ac->schema->loaded_usn; data->cached_schema_metadata_usn = ac->schema->metadata_usn; return LDB_SUCCESS; diff --git a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c index 0c94b2aa6b8..1db23927be0 100644 --- a/source4/dsdb/samdb/ldb_modules/repl_meta_data.c +++ b/source4/dsdb/samdb/ldb_modules/repl_meta_data.c @@ -565,6 +565,23 @@ static int replmd_notify_store(struct ldb_module *module, struct ldb_request *pa ldb_dn_get_linearized(modified_partition->dn))); return ret; } + + if (ldb_dn_compare(modified_partition->dn, + replmd_private->schema_dn) == 0) { + struct ldb_result *ext_res; + ret = dsdb_module_extended(module, + replmd_private->schema_dn, + &ext_res, + DSDB_EXTENDED_SCHEMA_UPDATE_NOW_OID, + ext_res, + DSDB_FLAG_NEXT_MODULE, + parent); + if (ret != LDB_SUCCESS) { + return ret; + } + talloc_free(ext_res); + } + DLIST_REMOVE(replmd_private->ncs, modified_partition); talloc_free(modified_partition); } diff --git a/source4/dsdb/samdb/ldb_modules/schema_load.c b/source4/dsdb/samdb/ldb_modules/schema_load.c index 0672b5c62d0..3f5cd2c62c2 100644 --- a/source4/dsdb/samdb/ldb_modules/schema_load.c +++ b/source4/dsdb/samdb/ldb_modules/schema_load.c @@ -35,6 +35,7 @@ #include "system/filesys.h" struct schema_load_private_data { + struct ldb_module *module; bool in_transaction; struct tdb_wrap *metadata; uint64_t schema_seq_num_cache; @@ -43,7 +44,6 @@ struct schema_load_private_data { static int dsdb_schema_from_db(struct ldb_module *module, TALLOC_CTX *mem_ctx, - uint64_t current_usn, uint64_t schema_seq_num, struct dsdb_schema **schema); @@ -107,22 +107,25 @@ static int schema_metadata_open(struct ldb_module *module) return LDB_SUCCESS; } -static int schema_metadata_get_uint64(struct ldb_module *module, +static int schema_metadata_get_uint64(struct schema_load_private_data *data, const char *key, uint64_t *value, uint64_t default_value) { - struct schema_load_private_data *data = talloc_get_type(ldb_module_get_private(module), struct schema_load_private_data); struct tdb_context *tdb; TDB_DATA tdb_key, tdb_data; char *value_str; TALLOC_CTX *tmp_ctx; int tdb_seqnum; - if (!data || !data->metadata) { + if (!data) { *value = default_value; return LDB_SUCCESS; } + if (!data->metadata) { + return LDB_ERR_OPERATIONS_ERROR; + } + tdb_seqnum = tdb_get_seqnum(data->metadata->tdb); if (tdb_seqnum == data->tdb_seqnum) { *value = data->schema_seq_num_cache; @@ -131,7 +134,7 @@ static int schema_metadata_get_uint64(struct ldb_module *module, tmp_ctx = talloc_new(NULL); if (tmp_ctx == NULL) { - return ldb_module_oom(module); + return ldb_module_oom(data->module); } tdb = data->metadata->tdb; @@ -147,7 +150,7 @@ static int schema_metadata_get_uint64(struct ldb_module *module, return LDB_SUCCESS; } else { talloc_free(tmp_ctx); - return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + return ldb_module_error(data->module, LDB_ERR_OPERATIONS_ERROR, tdb_errorstr(tdb)); } } @@ -156,7 +159,7 @@ static int schema_metadata_get_uint64(struct ldb_module *module, if (value_str == NULL) { SAFE_FREE(tdb_data.dptr); talloc_free(tmp_ctx); - return ldb_module_oom(module); + return ldb_module_oom(data->module); } /* @@ -179,12 +182,10 @@ static struct dsdb_schema *dsdb_schema_refresh(struct ldb_module *module, struct struct dsdb_schema *schema, bool is_global_schema) { TALLOC_CTX *mem_ctx; - uint64_t current_usn, schema_seq_num = 0; + uint64_t schema_seq_num = 0; int ret; struct ldb_context *ldb = ldb_module_get_ctx(module); struct dsdb_schema *new_schema; - struct ldb_dn *schema_dn = ldb_get_schema_basedn(ldb); - time_t ts, lastts; struct schema_load_private_data *private_data = talloc_get_type(ldb_module_get_private(module), struct schema_load_private_data); if (!private_data) { @@ -212,39 +213,31 @@ static struct dsdb_schema *dsdb_schema_refresh(struct ldb_module *module, struct * continue to hit the database to get the highest USN. */ - ret = schema_metadata_get_uint64(module, DSDB_METADATA_SCHEMA_SEQ_NUM, &schema_seq_num, 0); + ret = schema_metadata_get_uint64(private_data, DSDB_METADATA_SCHEMA_SEQ_NUM, &schema_seq_num, 0); if (schema != NULL) { - lastts = schema->last_refresh; - ts = time(NULL); - if (lastts > (ts - schema->refresh_interval)) { - DEBUG(11, ("Less than %d seconds since last reload, " - "returning cached version ts = %d\n", - (int)schema->refresh_interval, - (int)lastts)); - TALLOC_FREE(mem_ctx); - return schema; - } - if (ret == LDB_SUCCESS) { - schema->metadata_usn = schema_seq_num; + if (schema->metadata_usn == schema_seq_num) { + TALLOC_FREE(mem_ctx); + return schema; + } else { + DEBUG(3, ("Schema refresh needed %lld != %lld\n", + (unsigned long long)schema->metadata_usn, + (unsigned long long)schema_seq_num)); + } } else { /* From an old provision it can happen that the tdb didn't exists yet */ DEBUG(0, ("Error while searching for the schema usn in the metadata ignoring: %d:%s:%s\n", ret, ldb_strerror(ret), ldb_errstring(ldb))); - schema->metadata_usn = 0; + TALLOC_FREE(mem_ctx); + return schema; } - schema->last_refresh = ts; - - } - - ret = dsdb_module_load_partition_usn(module, schema_dn, ¤t_usn, NULL, NULL); - if (ret != LDB_SUCCESS || (schema && (current_usn == schema->loaded_usn))) { - TALLOC_FREE(mem_ctx); - return schema; + } else { + DEBUG(3, ("Initial schema load needed, as we have no existing schema, seq_num: %lld\n", + (unsigned long long)schema_seq_num)); } - ret = dsdb_schema_from_db(module, mem_ctx, current_usn, schema_seq_num, &new_schema); + ret = dsdb_schema_from_db(module, mem_ctx, schema_seq_num, &new_schema); if (ret != LDB_SUCCESS) { ldb_debug_set(ldb, LDB_DEBUG_FATAL, "dsdb_schema_from_db() failed: %d:%s: %s", @@ -275,7 +268,6 @@ static struct dsdb_schema *dsdb_schema_refresh(struct ldb_module *module, struct static int dsdb_schema_from_db(struct ldb_module *module, TALLOC_CTX *mem_ctx, - uint64_t current_usn, uint64_t schema_seq_num, struct dsdb_schema **schema) { @@ -346,9 +338,7 @@ static int dsdb_schema_from_db(struct ldb_module *module, goto failed; } - (*schema)->loaded_usn = current_usn; (*schema)->metadata_usn = schema_seq_num; - (*schema)->last_refresh = time(NULL); talloc_steal(mem_ctx, *schema); @@ -368,13 +358,14 @@ static int schema_load_init(struct ldb_module *module) struct ldb_context *ldb = ldb_module_get_ctx(module); struct dsdb_schema *schema; void *readOnlySchema; - int ret; + int ret, metadata_ret; private_data = talloc_zero(module, struct schema_load_private_data); if (private_data == NULL) { return ldb_oom(ldb); } - + private_data->module = module; + ldb_module_set_private(module, private_data); ret = ldb_next_init(module); @@ -384,10 +375,12 @@ static int schema_load_init(struct ldb_module *module) schema = dsdb_get_schema(ldb, NULL); + metadata_ret = schema_metadata_open(module); + /* We might already have a schema */ if (schema != NULL) { - /* Hook up the refresh function */ - if (dsdb_uses_global_schema(ldb)) { + /* If we have the metadata.tdb, then hook up the refresh function */ + if (metadata_ret == LDB_SUCCESS && dsdb_uses_global_schema(ldb)) { ret = dsdb_set_schema_refresh_function(ldb, dsdb_schema_refresh, module); if (ret != LDB_SUCCESS) { @@ -408,7 +401,7 @@ static int schema_load_init(struct ldb_module *module) * have to update the backend server schema too */ if (readOnlySchema != NULL) { struct dsdb_schema *new_schema; - ret = dsdb_schema_from_db(module, private_data, 0, 0, &new_schema); + ret = dsdb_schema_from_db(module, private_data, 0, &new_schema); if (ret != LDB_SUCCESS) { ldb_debug_set(ldb, LDB_DEBUG_FATAL, "schema_load_init: dsdb_schema_from_db() failed: %d:%s: %s", @@ -425,7 +418,7 @@ static int schema_load_init(struct ldb_module *module) return ret; } - } else { + } else if (metadata_ret == LDB_SUCCESS) { ret = dsdb_set_schema_refresh_function(ldb, dsdb_schema_refresh, module); if (ret != LDB_SUCCESS) { @@ -434,6 +427,10 @@ static int schema_load_init(struct ldb_module *module) ret, ldb_strerror(ret), ldb_errstring(ldb)); return ret; } + } else { + ldb_debug_set(ldb, LDB_DEBUG_FATAL, + "schema_load_init: failed to open metadata.tdb"); + return metadata_ret; } schema = dsdb_get_schema(ldb, NULL); @@ -450,25 +447,10 @@ static int schema_load_init(struct ldb_module *module) static int schema_search(struct ldb_module *module, struct ldb_request *req) { - struct dsdb_schema *schema; struct ldb_context *ldb = ldb_module_get_ctx(module); - uint64_t value; - int ret; - struct schema_load_private_data *private_data = - talloc_get_type(ldb_module_get_private(module), struct schema_load_private_data); - schema = dsdb_get_schema(ldb, NULL); - if (schema && private_data && !private_data->in_transaction) { - ret = schema_metadata_get_uint64(module, DSDB_METADATA_SCHEMA_SEQ_NUM, &value, 0); - if (ret == LDB_SUCCESS && schema->metadata_usn < value) { - /* The usn of the schema was changed in the metadata, - * this indicate that another process has modified the schema and - * that a reload is needed. - */ - schema->last_refresh = 0; - schema = dsdb_get_schema(ldb, NULL); - } - } + /* Try the schema refresh now */ + dsdb_get_schema(ldb, NULL); return ldb_next_request(module, req); } @@ -477,23 +459,15 @@ static int schema_load_start_transaction(struct ldb_module *module) { struct schema_load_private_data *private_data = talloc_get_type(ldb_module_get_private(module), struct schema_load_private_data); - struct dsdb_schema *schema; struct ldb_context *ldb = ldb_module_get_ctx(module); - uint64_t value; - int ret; + struct dsdb_schema *schema; + /* Try the schema refresh now */ schema = dsdb_get_schema(ldb, NULL); - if (!private_data->metadata) { - schema_metadata_open(module); - } - ret = schema_metadata_get_uint64(module, DSDB_METADATA_SCHEMA_SEQ_NUM, &value, 0); - if (ret == LDB_SUCCESS && schema->metadata_usn < value) { - /* The usn of the schema was changed in the metadata, - * this indicate that another process has modified the schema and - * that a reload is needed. - */ - schema->last_refresh = 0; - schema = dsdb_get_schema(ldb, NULL); + if (schema == NULL) { + ldb_debug_set(ldb, LDB_DEBUG_FATAL, + "schema_load_init: dsdb_get_schema failed"); + return LDB_ERR_OPERATIONS_ERROR; } private_data->in_transaction = true; @@ -524,7 +498,6 @@ static int schema_load_extended(struct ldb_module *module, struct ldb_request *r { time_t *lastts; struct ldb_context *ldb = ldb_module_get_ctx(module); - struct dsdb_schema *schema; if (strcmp(req->op.extended.oid, DSDB_EXTENDED_SCHEMA_UPDATE_NOW_OID) != 0) { return ldb_next_request(module, req); @@ -533,9 +506,8 @@ static int schema_load_extended(struct ldb_module *module, struct ldb_request *r if (!lastts) { lastts = talloc(ldb, time_t); } - schema = dsdb_get_schema(ldb, NULL); /* Force a refresh */ - schema->last_refresh = 0; + dsdb_get_schema(ldb, NULL); *lastts = 0; ldb_set_opaque(ldb, DSDB_OPAQUE_LAST_SCHEMA_UPDATE_MSG_OPAQUE_NAME, lastts); diff --git a/source4/dsdb/schema/schema.h b/source4/dsdb/schema/schema.h index 457d986e3d0..6c3581bb5e4 100644 --- a/source4/dsdb/schema/schema.h +++ b/source4/dsdb/schema/schema.h @@ -214,9 +214,6 @@ struct dsdb_schema { */ const char *schema_info; - /* We can also tell the schema version from the USN on the partition */ - uint64_t loaded_usn; - struct dsdb_attribute *attributes; struct dsdb_class *classes; @@ -251,8 +248,6 @@ struct dsdb_schema { /* Was this schema loaded from ldb (if so, then we will reload it when we detect a change in ldb) */ bool refresh_in_progress; time_t ts_last_change; - time_t last_refresh; - time_t refresh_interval; /* This 'opaque' is stored in the metadata and is used to check if the currently * loaded schema needs a reload because another process has signaled that it has been * requested to reload the schema (either due through DRS or via the schemaUpdateNow). diff --git a/source4/dsdb/schema/schema_init.c b/source4/dsdb/schema/schema_init.c index 2e6177346c7..16f213d2fe2 100644 --- a/source4/dsdb/schema/schema_init.c +++ b/source4/dsdb/schema/schema_init.c @@ -39,7 +39,6 @@ struct dsdb_schema *dsdb_new_schema(TALLOC_CTX *mem_ctx) if (!schema) { return NULL; } - schema->refresh_interval = 120; return schema; } @@ -88,8 +87,6 @@ struct dsdb_schema *dsdb_schema_copy_shallow(TALLOC_CTX *mem_ctx, } schema_copy->num_attributes = schema->num_attributes; - schema_copy->refresh_interval = schema->refresh_interval; - /* rebuild indexes */ ret = dsdb_setup_sorted_accessors(ldb, schema_copy); if (ret != LDB_SUCCESS) { @@ -935,9 +932,6 @@ int dsdb_schema_from_ldb_results(TALLOC_CTX *mem_ctx, struct ldb_context *ldb, if (lp_opaque) { struct loadparm_context *lp_ctx = talloc_get_type_abort(lp_opaque, struct loadparm_context); - schema->refresh_interval = lpcfg_parm_int(lp_ctx, NULL, "dsdb", "schema_reload_interval", schema->refresh_interval); - lp_ctx = talloc_get_type(ldb_get_opaque(ldb, "loadparm"), - struct loadparm_context); schema->fsmo.update_allowed = lpcfg_parm_bool(lp_ctx, NULL, "dsdb", "schema update allowed", false); -- 2.34.1