lib/ntdb/ntdb.c

   1  /*
   2    Trivial Database 2: fetch, store and misc routines.
   3    Copyright (C) Rusty Russell 2010
   4
   5    This library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 3 of the License, or (at your option) any later version.
   9
  10    This library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with this library; if not, see <http://www.gnu.org/licenses/>.
  17 */
  18 #include "private.h"
  19 #ifndef HAVE_LIBREPLACE
  20 #include <ccan/asprintf/asprintf.h>
  21 #include <stdarg.h>
  22 #endif
  23
  24 static enum NTDB_ERROR update_rec_hdr(struct ntdb_context *ntdb,
  25                                      ntdb_off_t off,
  26                                      ntdb_len_t keylen,
  27                                      ntdb_len_t datalen,
  28                                      struct ntdb_used_record *rec)
  29 {
  30         uint64_t dataroom = rec_data_length(rec) + rec_extra_padding(rec);
  31         enum NTDB_ERROR ecode;
  32
  33         ecode = set_header(ntdb, rec, NTDB_USED_MAGIC, keylen, datalen,
  34                            keylen + dataroom);
  35         if (ecode == NTDB_SUCCESS) {
  36                 ecode = ntdb_write_convert(ntdb, off, rec, sizeof(*rec));
  37         }
  38         return ecode;
  39 }
  40
  41 static enum NTDB_ERROR replace_data(struct ntdb_context *ntdb,
  42                                    struct hash_info *h,
  43                                    NTDB_DATA key, NTDB_DATA dbuf,
  44                                    ntdb_off_t old_off, ntdb_len_t old_room,
  45                                    bool growing)
  46 {
  47         ntdb_off_t new_off;
  48         enum NTDB_ERROR ecode;
  49
  50         /* Allocate a new record. */
  51         new_off = alloc(ntdb, key.dsize, dbuf.dsize, NTDB_USED_MAGIC, growing);
  52         if (NTDB_OFF_IS_ERR(new_off)) {
  53                 return NTDB_OFF_TO_ERR(new_off);
  54         }
  55
  56         /* We didn't like the existing one: remove it. */
  57         if (old_off) {
  58                 ntdb->stats.frees++;
  59                 ecode = add_free_record(ntdb, old_off,
  60                                         sizeof(struct ntdb_used_record)
  61                                         + key.dsize + old_room,
  62                                         NTDB_LOCK_WAIT, true);
  63                 if (ecode == NTDB_SUCCESS)
  64                         ecode = replace_in_hash(ntdb, h, new_off);
  65         } else {
  66                 ecode = add_to_hash(ntdb, h, new_off);
  67         }
  68         if (ecode != NTDB_SUCCESS) {
  69                 return ecode;
  70         }
  71
  72         new_off += sizeof(struct ntdb_used_record);
  73         ecode = ntdb->io->twrite(ntdb, new_off, key.dptr, key.dsize);
  74         if (ecode != NTDB_SUCCESS) {
  75                 return ecode;
  76         }
  77
  78         new_off += key.dsize;
  79         ecode = ntdb->io->twrite(ntdb, new_off, dbuf.dptr, dbuf.dsize);
  80         if (ecode != NTDB_SUCCESS) {
  81                 return ecode;
  82         }
  83
  84         if (ntdb->flags & NTDB_SEQNUM)
  85                 ntdb_inc_seqnum(ntdb);
  86
  87         return NTDB_SUCCESS;
  88 }
  89
  90 static enum NTDB_ERROR update_data(struct ntdb_context *ntdb,
  91                                   ntdb_off_t off,
  92                                   NTDB_DATA dbuf,
  93                                   ntdb_len_t extra)
  94 {
  95         enum NTDB_ERROR ecode;
  96
  97         ecode = ntdb->io->twrite(ntdb, off, dbuf.dptr, dbuf.dsize);
  98         if (ecode == NTDB_SUCCESS && extra) {
  99                 /* Put a zero in; future versions may append other data. */
 100                 ecode = ntdb->io->twrite(ntdb, off + dbuf.dsize, "", 1);
 101         }
 102         if (ntdb->flags & NTDB_SEQNUM)
 103                 ntdb_inc_seqnum(ntdb);
 104
 105         return ecode;
 106 }
 107
 108 _PUBLIC_ enum NTDB_ERROR ntdb_store(struct ntdb_context *ntdb,
 109                          NTDB_DATA key, NTDB_DATA dbuf, int flag)
 110 {
 111         struct hash_info h;
 112         ntdb_off_t off;
 113         ntdb_len_t old_room = 0;
 114         struct ntdb_used_record rec;
 115         enum NTDB_ERROR ecode;
 116
 117         off = find_and_lock(ntdb, key, F_WRLCK, &h, &rec);
 118         if (NTDB_OFF_IS_ERR(off)) {
 119                 return NTDB_OFF_TO_ERR(off);
 120         }
 121
 122         /* Now we have lock on this hash bucket. */
 123         if (flag == NTDB_INSERT) {
 124                 if (off) {
 125                         ecode = NTDB_ERR_EXISTS;
 126                         goto out;
 127                 }
 128         } else {
 129                 if (off) {
 130                         old_room = rec_data_length(&rec)
 131                                 + rec_extra_padding(&rec);
 132                         if (old_room >= dbuf.dsize) {
 133                                 /* Can modify in-place.  Easy! */
 134                                 ecode = update_rec_hdr(ntdb, off,
 135                                                        key.dsize, dbuf.dsize,
 136                                                        &rec);
 137                                 if (ecode != NTDB_SUCCESS) {
 138                                         goto out;
 139                                 }
 140                                 ecode = update_data(ntdb,
 141                                                     off + sizeof(rec)
 142                                                     + key.dsize, dbuf,
 143                                                     old_room - dbuf.dsize);
 144                                 if (ecode != NTDB_SUCCESS) {
 145                                         goto out;
 146                                 }
 147                                 ntdb_unlock_hash(ntdb, h.h, F_WRLCK);
 148                                 return NTDB_SUCCESS;
 149                         }
 150                 } else {
 151                         if (flag == NTDB_MODIFY) {
 152                                 /* if the record doesn't exist and we
 153                                    are in NTDB_MODIFY mode then we should fail
 154                                    the store */
 155                                 ecode = NTDB_ERR_NOEXIST;
 156                                 goto out;
 157                         }
 158                 }
 159         }
 160
 161         /* If we didn't use the old record, this implies we're growing. */
 162         ecode = replace_data(ntdb, &h, key, dbuf, off, old_room, off);
 163 out:
 164         ntdb_unlock_hash(ntdb, h.h, F_WRLCK);
 165         return ecode;
 166 }
 167
 168 _PUBLIC_ enum NTDB_ERROR ntdb_append(struct ntdb_context *ntdb,
 169                           NTDB_DATA key, NTDB_DATA dbuf)
 170 {
 171         struct hash_info h;
 172         ntdb_off_t off;
 173         struct ntdb_used_record rec;
 174         ntdb_len_t old_room = 0, old_dlen;
 175         unsigned char *newdata;
 176         NTDB_DATA new_dbuf;
 177         enum NTDB_ERROR ecode;
 178
 179         off = find_and_lock(ntdb, key, F_WRLCK, &h, &rec);
 180         if (NTDB_OFF_IS_ERR(off)) {
 181                 return NTDB_OFF_TO_ERR(off);
 182         }
 183
 184         if (off) {
 185                 old_dlen = rec_data_length(&rec);
 186                 old_room = old_dlen + rec_extra_padding(&rec);
 187
 188                 /* Fast path: can append in place. */
 189                 if (rec_extra_padding(&rec) >= dbuf.dsize) {
 190                         ecode = update_rec_hdr(ntdb, off, key.dsize,
 191                                                old_dlen + dbuf.dsize, &rec);
 192                         if (ecode != NTDB_SUCCESS) {
 193                                 goto out;
 194                         }
 195
 196                         off += sizeof(rec) + key.dsize + old_dlen;
 197                         ecode = update_data(ntdb, off, dbuf,
 198                                             rec_extra_padding(&rec));
 199                         goto out;
 200                 }
 201
 202                 /* Slow path. */
 203                 newdata = ntdb->alloc_fn(ntdb, key.dsize + old_dlen + dbuf.dsize,
 204                                      ntdb->alloc_data);
 205                 if (!newdata) {
 206                         ecode = ntdb_logerr(ntdb, NTDB_ERR_OOM, NTDB_LOG_ERROR,
 207                                            "ntdb_append:"
 208                                            " failed to allocate %zu bytes",
 209                                            (size_t)(key.dsize + old_dlen
 210                                                     + dbuf.dsize));
 211                         goto out;
 212                 }
 213                 ecode = ntdb->io->tread(ntdb, off + sizeof(rec) + key.dsize,
 214                                        newdata, old_dlen);
 215                 if (ecode != NTDB_SUCCESS) {
 216                         goto out_free_newdata;
 217                 }
 218                 memcpy(newdata + old_dlen, dbuf.dptr, dbuf.dsize);
 219                 new_dbuf.dptr = newdata;
 220                 new_dbuf.dsize = old_dlen + dbuf.dsize;
 221         } else {
 222                 newdata = NULL;
 223                 new_dbuf = dbuf;
 224         }
 225
 226         /* If they're using ntdb_append(), it implies they're growing record. */
 227         ecode = replace_data(ntdb, &h, key, new_dbuf, off, old_room, true);
 228
 229 out_free_newdata:
 230         ntdb->free_fn(newdata, ntdb->alloc_data);
 231 out:
 232         ntdb_unlock_hash(ntdb, h.h, F_WRLCK);
 233         return ecode;
 234 }
 235
 236 _PUBLIC_ enum NTDB_ERROR ntdb_fetch(struct ntdb_context *ntdb, NTDB_DATA key,
 237                          NTDB_DATA *data)
 238 {
 239         ntdb_off_t off;
 240         struct ntdb_used_record rec;
 241         struct hash_info h;
 242         enum NTDB_ERROR ecode;
 243
 244         off = find_and_lock(ntdb, key, F_RDLCK, &h, &rec);
 245         if (NTDB_OFF_IS_ERR(off)) {
 246                 return NTDB_OFF_TO_ERR(off);
 247         }
 248
 249         if (!off) {
 250                 ecode = NTDB_ERR_NOEXIST;
 251         } else {
 252                 data->dsize = rec_data_length(&rec);
 253                 data->dptr = ntdb_alloc_read(ntdb, off + sizeof(rec) + key.dsize,
 254                                             data->dsize);
 255                 if (NTDB_PTR_IS_ERR(data->dptr)) {
 256                         ecode = NTDB_PTR_ERR(data->dptr);
 257                 } else
 258                         ecode = NTDB_SUCCESS;
 259         }
 260
 261         ntdb_unlock_hash(ntdb, h.h, F_RDLCK);
 262         return ecode;
 263 }
 264
 265 _PUBLIC_ bool ntdb_exists(struct ntdb_context *ntdb, NTDB_DATA key)
 266 {
 267         ntdb_off_t off;
 268         struct ntdb_used_record rec;
 269         struct hash_info h;
 270
 271         off = find_and_lock(ntdb, key, F_RDLCK, &h, &rec);
 272         if (NTDB_OFF_IS_ERR(off)) {
 273                 return false;
 274         }
 275         ntdb_unlock_hash(ntdb, h.h, F_RDLCK);
 276
 277         return off ? true : false;
 278 }
 279
 280 _PUBLIC_ enum NTDB_ERROR ntdb_delete(struct ntdb_context *ntdb, NTDB_DATA key)
 281 {
 282         ntdb_off_t off;
 283         struct ntdb_used_record rec;
 284         struct hash_info h;
 285         enum NTDB_ERROR ecode;
 286
 287         off = find_and_lock(ntdb, key, F_WRLCK, &h, &rec);
 288         if (NTDB_OFF_IS_ERR(off)) {
 289                 return NTDB_OFF_TO_ERR(off);
 290         }
 291
 292         if (!off) {
 293                 ecode = NTDB_ERR_NOEXIST;
 294                 goto unlock;
 295         }
 296
 297         ecode = delete_from_hash(ntdb, &h);
 298         if (ecode != NTDB_SUCCESS) {
 299                 goto unlock;
 300         }
 301
 302         /* Free the deleted entry. */
 303         ntdb->stats.frees++;
 304         ecode = add_free_record(ntdb, off,
 305                                 sizeof(struct ntdb_used_record)
 306                                 + rec_key_length(&rec)
 307                                 + rec_data_length(&rec)
 308                                 + rec_extra_padding(&rec),
 309                                 NTDB_LOCK_WAIT, true);
 310
 311         if (ntdb->flags & NTDB_SEQNUM)
 312                 ntdb_inc_seqnum(ntdb);
 313
 314 unlock:
 315         ntdb_unlock_hash(ntdb, h.h, F_WRLCK);
 316         return ecode;
 317 }
 318
 319 _PUBLIC_ unsigned int ntdb_get_flags(struct ntdb_context *ntdb)
 320 {
 321         return ntdb->flags;
 322 }
 323
 324 static bool inside_transaction(const struct ntdb_context *ntdb)
 325 {
 326         return ntdb->transaction != NULL;
 327 }
 328
 329 static bool readonly_changable(struct ntdb_context *ntdb, const char *caller)
 330 {
 331         if (inside_transaction(ntdb)) {
 332                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 333                             "%s: can't change"
 334                             " NTDB_RDONLY inside transaction",
 335                             caller);
 336                 return false;
 337         }
 338         return true;
 339 }
 340
 341 _PUBLIC_ void ntdb_add_flag(struct ntdb_context *ntdb, unsigned flag)
 342 {
 343         if (ntdb->flags & NTDB_INTERNAL) {
 344                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 345                             "ntdb_add_flag: internal db");
 346                 return;
 347         }
 348         switch (flag) {
 349         case NTDB_NOLOCK:
 350                 ntdb->flags |= NTDB_NOLOCK;
 351                 break;
 352         case NTDB_NOMMAP:
 353                 ntdb->flags |= NTDB_NOMMAP;
 354 #ifndef HAVE_INCOHERENT_MMAP
 355                 ntdb_munmap(ntdb->file);
 356 #endif
 357                 break;
 358         case NTDB_NOSYNC:
 359                 ntdb->flags |= NTDB_NOSYNC;
 360                 break;
 361         case NTDB_SEQNUM:
 362                 ntdb->flags |= NTDB_SEQNUM;
 363                 break;
 364         case NTDB_ALLOW_NESTING:
 365                 ntdb->flags |= NTDB_ALLOW_NESTING;
 366                 break;
 367         case NTDB_RDONLY:
 368                 if (readonly_changable(ntdb, "ntdb_add_flag"))
 369                         ntdb->flags |= NTDB_RDONLY;
 370                 break;
 371         default:
 372                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 373                             "ntdb_add_flag: Unknown flag %u", flag);
 374         }
 375 }
 376
 377 _PUBLIC_ void ntdb_remove_flag(struct ntdb_context *ntdb, unsigned flag)
 378 {
 379         if (ntdb->flags & NTDB_INTERNAL) {
 380                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 381                             "ntdb_remove_flag: internal db");
 382                 return;
 383         }
 384         switch (flag) {
 385         case NTDB_NOLOCK:
 386                 ntdb->flags &= ~NTDB_NOLOCK;
 387                 break;
 388         case NTDB_NOMMAP:
 389                 ntdb->flags &= ~NTDB_NOMMAP;
 390 #ifndef HAVE_INCOHERENT_MMAP
 391                 /* If mmap incoherent, we were mmaping anyway. */
 392                 ntdb_mmap(ntdb);
 393 #endif
 394                 break;
 395         case NTDB_NOSYNC:
 396                 ntdb->flags &= ~NTDB_NOSYNC;
 397                 break;
 398         case NTDB_SEQNUM:
 399                 ntdb->flags &= ~NTDB_SEQNUM;
 400                 break;
 401         case NTDB_ALLOW_NESTING:
 402                 ntdb->flags &= ~NTDB_ALLOW_NESTING;
 403                 break;
 404         case NTDB_RDONLY:
 405                 if ((ntdb->open_flags & O_ACCMODE) == O_RDONLY) {
 406                         ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 407                                     "ntdb_remove_flag: can't"
 408                                     " remove NTDB_RDONLY on ntdb"
 409                                     " opened with O_RDONLY");
 410                         break;
 411                 }
 412                 if (readonly_changable(ntdb, "ntdb_remove_flag"))
 413                         ntdb->flags &= ~NTDB_RDONLY;
 414                 break;
 415         default:
 416                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
 417                             "ntdb_remove_flag: Unknown flag %u",
 418                             flag);
 419         }
 420 }
 421
 422 _PUBLIC_ const char *ntdb_errorstr(enum NTDB_ERROR ecode)
 423 {
 424         /* Gcc warns if you miss a case in the switch, so use that. */
 425         switch (NTDB_ERR_TO_OFF(ecode)) {
 426         case NTDB_ERR_TO_OFF(NTDB_SUCCESS): return "Success";
 427         case NTDB_ERR_TO_OFF(NTDB_ERR_CORRUPT): return "Corrupt database";
 428         case NTDB_ERR_TO_OFF(NTDB_ERR_IO): return "IO Error";
 429         case NTDB_ERR_TO_OFF(NTDB_ERR_LOCK): return "Locking error";
 430         case NTDB_ERR_TO_OFF(NTDB_ERR_OOM): return "Out of memory";
 431         case NTDB_ERR_TO_OFF(NTDB_ERR_EXISTS): return "Record exists";
 432         case NTDB_ERR_TO_OFF(NTDB_ERR_EINVAL): return "Invalid parameter";
 433         case NTDB_ERR_TO_OFF(NTDB_ERR_NOEXIST): return "Record does not exist";
 434         case NTDB_ERR_TO_OFF(NTDB_ERR_RDONLY): return "write not permitted";
 435         }
 436         return "Invalid error code";
 437 }
 438
 439 enum NTDB_ERROR COLD ntdb_logerr(struct ntdb_context *ntdb,
 440                                enum NTDB_ERROR ecode,
 441                                enum ntdb_log_level level,
 442                                const char *fmt, ...)
 443 {
 444         char *message;
 445         va_list ap;
 446         size_t len;
 447         /* ntdb_open paths care about errno, so save it. */
 448         int saved_errno = errno;
 449
 450         if (!ntdb->log_fn)
 451                 return ecode;
 452
 453         va_start(ap, fmt);
 454         len = vsnprintf(NULL, 0, fmt, ap);
 455         va_end(ap);
 456
 457         message = ntdb->alloc_fn(ntdb, len + 1, ntdb->alloc_data);
 458         if (!message) {
 459                 ntdb->log_fn(ntdb, NTDB_LOG_ERROR, NTDB_ERR_OOM,
 460                             "out of memory formatting message:", ntdb->log_data);
 461                 ntdb->log_fn(ntdb, level, ecode, fmt, ntdb->log_data);
 462         } else {
 463                 va_start(ap, fmt);
 464                 vsnprintf(message, len+1, fmt, ap);
 465                 va_end(ap);
 466                 ntdb->log_fn(ntdb, level, ecode, message, ntdb->log_data);
 467                 ntdb->free_fn(message, ntdb->alloc_data);
 468         }
 469         errno = saved_errno;
 470         return ecode;
 471 }
 472
 473 _PUBLIC_ enum NTDB_ERROR ntdb_parse_record_(struct ntdb_context *ntdb,
 474                                  NTDB_DATA key,
 475                                  enum NTDB_ERROR (*parse)(NTDB_DATA k,
 476                                                          NTDB_DATA d,
 477                                                          void *data),
 478                                  void *data)
 479 {
 480         ntdb_off_t off;
 481         struct ntdb_used_record rec;
 482         struct hash_info h;
 483         enum NTDB_ERROR ecode;
 484
 485         off = find_and_lock(ntdb, key, F_RDLCK, &h, &rec);
 486         if (NTDB_OFF_IS_ERR(off)) {
 487                 return NTDB_OFF_TO_ERR(off);
 488         }
 489
 490         if (!off) {
 491                 ecode = NTDB_ERR_NOEXIST;
 492         } else {
 493                 const void *dptr;
 494                 dptr = ntdb_access_read(ntdb, off + sizeof(rec) + key.dsize,
 495                                        rec_data_length(&rec), false);
 496                 if (NTDB_PTR_IS_ERR(dptr)) {
 497                         ecode = NTDB_PTR_ERR(dptr);
 498                 } else {
 499                         NTDB_DATA d = ntdb_mkdata(dptr, rec_data_length(&rec));
 500
 501                         ecode = parse(key, d, data);
 502                         ntdb_access_release(ntdb, dptr);
 503                 }
 504         }
 505
 506         ntdb_unlock_hash(ntdb, h.h, F_RDLCK);
 507         return ecode;
 508 }
 509
 510 _PUBLIC_ const char *ntdb_name(const struct ntdb_context *ntdb)
 511 {
 512         return ntdb->name;
 513 }
 514
 515 _PUBLIC_ int64_t ntdb_get_seqnum(struct ntdb_context *ntdb)
 516 {
 517         return ntdb_read_off(ntdb, offsetof(struct ntdb_header, seqnum));
 518 }
 519
 520
 521 _PUBLIC_ int ntdb_fd(const struct ntdb_context *ntdb)
 522 {
 523         return ntdb->file->fd;
 524 }
 525
 526 struct traverse_state {
 527         enum NTDB_ERROR error;
 528         struct ntdb_context *dest_db;
 529 };
 530
 531 /*
 532   traverse function for repacking
 533  */
 534 static int repack_traverse(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA data,
 535                            struct traverse_state *state)
 536 {
 537         state->error = ntdb_store(state->dest_db, key, data, NTDB_INSERT);
 538         if (state->error != NTDB_SUCCESS) {
 539                 return -1;
 540         }
 541         return 0;
 542 }
 543
 544 _PUBLIC_ enum NTDB_ERROR ntdb_repack(struct ntdb_context *ntdb)
 545 {
 546         struct ntdb_context *tmp_db;
 547         struct traverse_state state;
 548
 549         state.error = ntdb_transaction_start(ntdb);
 550         if (state.error != NTDB_SUCCESS) {
 551                 return state.error;
 552         }
 553
 554         tmp_db = ntdb_open("tmpdb", NTDB_INTERNAL, O_RDWR|O_CREAT, 0, NULL);
 555         if (tmp_db == NULL) {
 556                 state.error = ntdb_logerr(ntdb, NTDB_ERR_OOM, NTDB_LOG_ERROR,
 557                                          __location__
 558                                          " Failed to create tmp_db");
 559                 ntdb_transaction_cancel(ntdb);
 560                 return state.error;
 561         }
 562
 563         state.dest_db = tmp_db;
 564         if (ntdb_traverse(ntdb, repack_traverse, &state) < 0) {
 565                 goto fail;
 566         }
 567
 568         state.error = ntdb_wipe_all(ntdb);
 569         if (state.error != NTDB_SUCCESS) {
 570                 goto fail;
 571         }
 572
 573         state.dest_db = ntdb;
 574         if (ntdb_traverse(tmp_db, repack_traverse, &state) < 0) {
 575                 goto fail;
 576         }
 577
 578         ntdb_close(tmp_db);
 579         return ntdb_transaction_commit(ntdb);
 580
 581 fail:
 582         ntdb_transaction_cancel(ntdb);
 583         ntdb_close(tmp_db);
 584         return state.error;
 585 }