diff --git a/src/allocator.h b/src/allocator.h index eb83b631..54c8b939 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -136,6 +136,16 @@ typedef platform_status (*alloc_fn)(allocator *al, typedef refcount (*dec_ref_fn)(allocator *al, uint64 addr, page_type type); typedef refcount (*generic_ref_fn)(allocator *al, uint64 addr); +/* + * Record one logical reference to addr while rebuilding a recovery map (see + * rc_allocator_mount_recovery()). The first reference to a given extent + * establishes its nonzero allocation floor; later references increment it + * normally. addr must be the base address of a non-reserved extent. + */ +typedef platform_status (*recovery_record_reference_fn)(allocator *al, + uint64 addr, + page_type type); + typedef platform_status (*get_super_addr_fn)(allocator *al, allocator_root_id spl_id, uint64 *addr); @@ -157,9 +167,10 @@ typedef struct allocator_ops { allocator_get_config_fn get_config; alloc_fn alloc; - generic_ref_fn inc_ref; - dec_ref_fn dec_ref; - generic_ref_fn get_ref; + generic_ref_fn inc_ref; + dec_ref_fn dec_ref; + generic_ref_fn get_ref; + recovery_record_reference_fn recovery_record_reference; alloc_super_addr_fn alloc_super_addr; get_super_addr_fn get_super_addr; @@ -211,6 +222,12 @@ allocator_get_refcount(allocator *al, uint64 addr) return al->ops->get_ref(al, addr); } +static inline platform_status +allocator_recovery_record_reference(allocator *al, uint64 addr, page_type type) +{ + return al->ops->recovery_record_reference(al, addr, type); +} + static inline platform_status allocator_get_super_addr(allocator *al, allocator_root_id spl_id, uint64 *addr) { diff --git a/src/blob.c b/src/blob.c index b6c8e084..f8c0c375 100644 --- a/src/blob.c +++ b/src/blob.c @@ -226,7 +226,6 @@ blob_page_iterator_get_curr(blob_page_iterator *iter, iter->cc, iter->fragment.addr, TRUE, PAGE_TYPE_BLOB); } cache_lock(iter->cc, iter->page); - cache_mark_dirty(iter->cc, iter->page); } } } @@ -327,7 +326,7 @@ blob_sync(cache *cc, slice sblob) if (!SUCCESS(rc)) { break; } - cache_page_sync(cc, itor.page, FALSE, PAGE_TYPE_BLOB); + cache_page_writeback(cc, itor.page, FALSE, PAGE_TYPE_BLOB); blob_page_iterator_advance_page(&itor); } diff --git a/src/btree.c b/src/btree.c index e93b0c50..9504aba3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1197,7 +1197,6 @@ btree_node_lock(cache *cc, // IN btree_node *node) // IN { cache_lock(cc, node->page); - cache_mark_dirty(cc, node->page); } static inline void @@ -1294,8 +1293,6 @@ btree_create(cache *cc, btree_init_hdr(cfg, root.hdr); - cache_mark_dirty(cc, root.page); - // If this btree is for a memtable then pin all pages belonging to it if (pinned) { cache_pin(cc, root.page); diff --git a/src/cache.h b/src/cache.h index bc239b7a..598df0f5 100644 --- a/src/cache.h +++ b/src/cache.h @@ -94,17 +94,6 @@ cache_config_extent_page(const cache_config *cfg, uint64 extent_addr, uint64 i) return extent_addr + i * cache_config_page_size(cfg); } -typedef void (*cache_generic_fn)(cache *cc); -typedef uint64 (*cache_generic_uint64_fn)(cache *cc); -typedef void (*page_generic_fn)(cache *cc, page_handle *page); - -typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); -typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); -typedef page_handle *(*page_get_fn)(cache *cc, - uint64 addr, - bool32 blocking, - page_type type); - #define PAGE_GET_ASYNC_STATE_BUFFER_SIZE (2048) typedef union page_get_async_state_payload { uint8 bytes[PAGE_GET_ASYNC_STATE_BUFFER_SIZE]; @@ -118,6 +107,11 @@ typedef struct page_get_async_state_buffer { page_get_async_state_payload payload; } page_get_async_state_buffer; +typedef void (*cache_generic_void_fn)(cache *cc); +typedef uint64 (*cache_generic_uint64_fn)(cache *cc); +typedef platform_status (*cache_generic_status_fn)(cache *cc); +typedef void (*page_generic_fn)(cache *cc, page_handle *page); + typedef void (*page_get_async_state_init_fn)(void *payload, cache *cc, uint64 addr, @@ -127,14 +121,20 @@ typedef void (*page_get_async_state_init_fn)(void *payload, typedef async_status (*page_get_async_fn)(void *payload); typedef page_handle *(*page_get_async_state_result_fn)(void *payload); +typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); +typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); +typedef page_handle *(*page_get_fn)(cache *cc, + uint64 addr, + bool32 blocking, + page_type type); typedef bool32 (*page_try_claim_fn)(cache *cc, page_handle *page); -typedef void (*page_sync_fn)(cache *cc, - page_handle *page, - bool32 is_blocking, - page_type type); -typedef void (*extent_sync_fn)(cache *cc, - uint64 addr, - uint64 *pages_outstanding); +typedef void (*page_writeback_fn)(cache *cc, + page_handle *page, + bool32 is_blocking, + page_type type); +typedef void (*extent_writeback_fn)(cache *cc, + uint64 addr, + uint64 *pages_outstanding); typedef void (*page_prefetch_fn)(cache *cc, uint64 addr, page_type type); typedef int (*evict_fn)(cache *cc, bool32 ignore_pinned); typedef bool32 (*page_addr_pred_fn)(cache *cc, uint64 addr); @@ -163,35 +163,36 @@ typedef struct cache_ops { page_get_async_fn page_get_async; page_get_async_state_result_fn page_get_async_result; - page_generic_fn page_unget; - page_try_claim_fn page_try_claim; - page_generic_fn page_unclaim; - page_generic_fn page_lock; - page_generic_fn page_unlock; - page_prefetch_fn page_prefetch; - page_prefetch_fn page_prefetch_page; - page_generic_fn page_mark_dirty; - page_generic_fn page_pin; - page_generic_fn page_unpin; - page_sync_fn page_sync; - extent_sync_fn extent_sync; - cache_generic_fn flush; - evict_fn evict; - cache_generic_fn cleanup; - page_addr_pred_fn in_use; - page_addr_fn assert_ungot; - cache_generic_fn assert_free; - validate_page_fn validate_page; - cache_present_fn cache_present; - cache_print_fn print; - cache_print_fn print_stats; - io_stats_fn io_stats; - cache_generic_fn reset_stats; - count_dirty_fn count_dirty; - page_get_read_ref_fn page_get_read_ref; - enable_sync_get_fn enable_sync_get; - get_allocator_fn get_allocator; - cache_config_fn get_config; + page_generic_fn page_unget; + page_try_claim_fn page_try_claim; + page_generic_fn page_unclaim; + page_generic_fn page_lock; + page_generic_fn page_unlock; + page_prefetch_fn page_prefetch; + page_prefetch_fn page_prefetch_page; + page_generic_fn page_pin; + page_generic_fn page_unpin; + page_writeback_fn page_writeback; + extent_writeback_fn extent_writeback; + cache_generic_void_fn flush; + cache_generic_status_fn writeback_dirty; + cache_generic_status_fn durable_barrier; + evict_fn evict; + cache_generic_void_fn cleanup; + page_addr_pred_fn in_use; + page_addr_fn assert_ungot; + cache_generic_void_fn assert_free; + validate_page_fn validate_page; + cache_present_fn cache_present; + cache_print_fn print; + cache_print_fn print_stats; + io_stats_fn io_stats; + cache_generic_void_fn reset_stats; + count_dirty_fn count_dirty; + page_get_read_ref_fn page_get_read_ref; + enable_sync_get_fn enable_sync_get; + get_allocator_fn get_allocator; + cache_config_fn get_config; } cache_ops; // To sub-class cache, make a cache your first field; @@ -366,8 +367,7 @@ cache_unclaim(cache *cc, page_handle *page) * * Blocks until outstanding read locks are released by other threads. * - * If you call this method, you almost certainly want to call - * cache_mark_dirty() immediately afterward. + * Acquiring the write lock marks the page dirty. *---------------------------------------------------------------------- */ static inline void @@ -427,27 +427,6 @@ cache_prefetch_page(cache *cc, uint64 addr, page_type type) return cc->ops->page_prefetch_page(cc, addr, type); } -/* - *---------------------------------------------------------------------- - * cache_mark_dirty - * - * Marks a page changed, to be written back. - * The caller had better have the write lock on the page via cache_lock() - * before changing its value. - * - * TODO This method should be removed; its effect should come automatically - * with the acquisition of a write lock. @robj reports lots of bugs - * due to forgetting to call this method. And we can't think of a case - * where we'd want the "optimization" of taking a write lock but then - * decide not to dirty it. - *---------------------------------------------------------------------- - */ -static inline void -cache_mark_dirty(cache *cc, page_handle *page) -{ - return cc->ops->page_mark_dirty(cc, page); -} - /* *---------------------------------------------------------------------- * cache_pin @@ -484,37 +463,44 @@ cache_unpin(cache *cc, page_handle *page) /* *----------------------------------------------------------------------------- - * cache_page_sync + * cache_page_writeback * - * Asynchronously writes the page back to disk. + * Issues writeback of the page to disk. This does NOT make the page durable; + * it only hands the write to the I/O layer (no device-cache flush). * - * This is used to sync log pages opportunistically. The current API doesn't - * inform the user when this happens. "It's not the ideal API." -- @aconway + * With is_blocking == FALSE the writeback is issued asynchronously and this + * returns without waiting for completion; there is no per-call way to observe + * when it finishes. With is_blocking == TRUE the page is written synchronously + * and the write has completed by the time this returns. *----------------------------------------------------------------------------- */ static inline void -cache_page_sync(cache *cc, - page_handle *page, - bool32 is_blocking, - page_type type) +cache_page_writeback(cache *cc, + page_handle *page, + bool32 is_blocking, + page_type type) { - return cc->ops->page_sync(cc, page, is_blocking, type); + return cc->ops->page_writeback(cc, page, is_blocking, type); } /* *----------------------------------------------------------------------------- - * cache_extent_sync + * cache_extent_writeback * - * Asynchronously syncs the extent beginning at addr. + * Asynchronously issues writeback of the extent beginning at addr. This does + * NOT make the extent durable; it only hands the writes to the I/O layer (no + * device-cache flush) and returns without waiting for completion. * * *pages_outstanding is immediately incremented by the number of pages * issued for writeback (the non-clean pages of the extent); as writebacks - * complete, *pages_outstanding is decremented atomically. + * complete, *pages_outstanding is decremented atomically. This counter is the + * only way to observe when the issued writebacks finish. * * Assumes pages_outstanding is an aligned uint64, so (on x86) the caller * can access it and observe its value atomically. * - * TODO: What happens if two callers call cache_extent_sync on the same extent? + * TODO: What happens if two callers call cache_extent_writeback on the same + * extent? * * All pages in the extent must be clean or cleanable. * The page may not be in writeback, loading, or locked, or claimed, otherwise @@ -522,9 +508,9 @@ cache_page_sync(cache *cc, *----------------------------------------------------------------------------- */ static inline void -cache_extent_sync(cache *cc, uint64 addr, uint64 *pages_outstanding) +cache_extent_writeback(cache *cc, uint64 addr, uint64 *pages_outstanding) { - cc->ops->extent_sync(cc, addr, pages_outstanding); + cc->ops->extent_writeback(cc, addr, pages_outstanding); } /* @@ -542,6 +528,35 @@ cache_flush(cache *cc) cc->ops->flush(cc); } +/* + *----------------------------------------------------------------------------- + * cache_writeback_dirty + * + * Issues and wait for completion of writebacks for all pages that are dirty + * but not locked at the time of the call. May writeback other pages, as well. + *----------------------------------------------------------------------------- + */ +static inline platform_status +cache_writeback_dirty(cache *cc) +{ + return cc->ops->writeback_dirty(cc); +} + +/* + *----------------------------------------------------------------------------- + * cache_durable_barrier + * + * Ensure that writeback completed before this call is durable across a power + * loss. Callers normally use this after cache_writeback_dirty(), and again + * after publishing a checkpoint superblock. + *----------------------------------------------------------------------------- + */ +static inline platform_status +cache_durable_barrier(cache *cc) +{ + return cc->ops->durable_barrier(cc); +} + /* *----------------------------------------------------------------------------- * cache_evict diff --git a/src/clockcache.c b/src/clockcache.c index b8197236..bd59647b 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -222,6 +222,50 @@ clockcache_test_flag(clockcache *cc, uint32 entry_number, entry_status flag) return flag & clockcache_get_status(cc, entry_number); } +/* + *-------------------------------------------------------------------------- + * Dirty-generation bookkeeping + * + * Each entry records the generation in which its current dirty interval began + * (0 when clean/free). A writeback fence takes a "cut" of the generation + * counter and drains every entry stamped at or below that cut. These + * transitions need no dedicated lock: clockcache_dirty_begin runs under the + * page's write lock, and a write lock cannot be held while CC_WRITEBACK is set + * (see clockcache_get_write), so a clean->dirty transition never races a + * writeback completion on the same entry. + *-------------------------------------------------------------------------- + */ +static void +clockcache_dirty_begin(clockcache *cc, uint32 entry_number) +{ + if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + debug_assert(entry->dirty_generation == 0); + // Stamp before clearing CC_CLEAN so a fence that observes the page dirty + // always sees a valid generation. clear_flag is a full barrier. + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + clockcache_clear_flag(cc, entry_number, CC_CLEAN); + } +} + +/* + * Mark an entry clean once its writeback has completed. CC_CLEAN must be set + * before CC_WRITEBACK is cleared: the intermediate CC_CLEAN|CC_WRITEBACK state + * is not cleanable, so no thread can start a duplicate writeback in the gap. + */ +static void +clockcache_dirty_complete_writeback(clockcache *cc, uint32 entry_number) +{ + clockcache_get_entry(cc, entry_number)->dirty_generation = 0; + debug_only uint32 was_clean = + clockcache_set_flag(cc, entry_number, CC_CLEAN); + debug_assert(!was_clean); + debug_only uint32 was_writeback = + clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); + debug_assert(was_writeback); +} + #ifdef RECORD_ACQUISITION_STACKS static void clockcache_record_backtrace(clockcache *cc, uint32 entry_number) @@ -785,8 +829,8 @@ clockcache_ok_to_writeback(clockcache *cc, bool32 with_access) { uint32 status = clockcache_get_status(cc, entry_number); - return ((status == CC_CLEANABLE1_STATUS) - || (with_access && status == CC_CLEANABLE2_STATUS)); + return (status == CC_CLEANABLE1_STATUS) + || (with_access && status == CC_CLEANABLE2_STATUS); } /* @@ -797,6 +841,13 @@ clockcache_ok_to_writeback(clockcache *cc, * status must be: * -- CC_CLEANABLE1_STATUS (= 0) // dirty * -- CC_CLEANABLE2_STATUS (= 0 | CC_ACCESSED) // dirty + * + * Returns FALSE only if the page is genuinely not writeback-able (locked, + * claimed, already in writeback, clean, ...). The CC_ACCESSED bit can flip + * (a reader sets it, the clock hand clears it) between the two + * compare-and- swaps, so we retry as long as the status remains one of the + * cleanable states rather than spuriously failing on a page that stayed + * cleanable. *---------------------------------------------------------------------- */ static inline bool32 @@ -811,19 +862,27 @@ clockcache_try_set_writeback(clockcache *cc, cc->cfg->page_capacity); volatile uint32 *status = &cc->entry[entry_number].status; - if (__sync_bool_compare_and_swap( - status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) - { - return TRUE; - } - - if (with_access - && __sync_bool_compare_and_swap( - status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) - { - return TRUE; + while (TRUE) { + uint32 cur = *status; + if (cur == CC_CLEANABLE1_STATUS) { + if (__sync_bool_compare_and_swap( + status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) + { + return TRUE; + } + } else if (with_access && cur == CC_CLEANABLE2_STATUS) { + if (__sync_bool_compare_and_swap( + status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) + { + return TRUE; + } + } else { + // Not a cleanable state: the page cannot be written back right now. + return FALSE; + } + // The CAS failed because the status changed under us. If it is still + // cleanable, retry; otherwise the next iteration returns FALSE. } - return FALSE; } typedef struct async_io_state { @@ -861,8 +920,6 @@ clockcache_write_callback(void *wbs) uint32 entry_number; clockcache_entry *entry; uint64 addr; - debug_only uint32 debug_status; - for (i = 0; i < count; i++) { entry_number = clockcache_data_to_entry_number(cc, (char *)iovec[i].iov_base); @@ -876,10 +933,7 @@ clockcache_write_callback(void *wbs) entry_number, addr); - debug_status = clockcache_set_flag(cc, entry_number, CC_CLEAN); - debug_assert(!debug_status); - debug_status = clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); - debug_assert(debug_status); + clockcache_dirty_complete_writeback(cc, entry_number); } if (state->outstanding_pages) { @@ -922,7 +976,7 @@ clockcache_abort_writeback_range(clockcache *cc, * they are not. *---------------------------------------------------------------------- */ -void +static platform_status clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) { uint32 entry_no, next_entry_no; @@ -932,6 +986,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) uint64 end_entry_no = start_entry_no + CC_ENTRIES_PER_BATCH; clockcache_entry *entry, *next_entry; + platform_status result = STATUS_OK; debug_assert((tid < MAX_THREADS), "Invalid tid=%lu\n", tid); debug_assert(cc != NULL); @@ -968,6 +1023,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY + && clockcache_ok_to_writeback(cc, next_entry_no, is_urgent) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); first_addr += page_size; end_addr = entry->page.disk_addr; @@ -981,6 +1037,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY + && clockcache_ok_to_writeback(cc, next_entry_no, is_urgent) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); @@ -991,6 +1048,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) "clockcache_batch_start_writeback: async_io_state allocation " "failed\n"); clockcache_abort_writeback_range(cc, first_addr, end_addr); + result = STATUS_NO_MEMORY; goto close_log; } @@ -1008,6 +1066,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) platform_status_to_string(rc)); clockcache_abort_writeback_range(cc, first_addr, end_addr); platform_free(PROCESS_PRIVATE_HEAP_ID, state); + result = rc; goto close_log; } @@ -1033,6 +1092,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) io_async_state_deinit(state->iostate); clockcache_abort_writeback_range(cc, first_addr, end_addr); platform_free(PROCESS_PRIVATE_HEAP_ID, state); + result = rc; goto close_log; } } @@ -1042,12 +1102,95 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) cc->stats[tid].writes_issued++; } - io_async_run(state->iostate); + // The IO layer must run writeback asynchronously and complete it via + // clockcache_write_callback (which does the dirty->clean bookkeeping). + // A synchronous completion would skip that callback and strand the + // pages in CC_WRITEBACK, so a broken contract is a fatal correctness + // error. + async_status arc = io_async_run(state->iostate); + platform_assert( + arc == ASYNC_STATUS_RUNNING, + "clockcache_batch_start_writeback: async writeback for addr %lu " + "completed synchronously; the IO layer must complete it via the " + "write callback", + first_addr); } } close_log: clockcache_close_log_stream(); + return result; +} + +/* + *-------------------------------------------------------------------------- + * clockcache_writeback_dirty -- + * + * Write back every page that was dirty when this call began, and wait for + * those writes (only those) to complete, so a subsequent durable_barrier can + * make them durable. + * + * We take a "cut" by incrementing the dirty generation: pages dirtied before + * now carry a generation <= cutoff and must be drained; pages dirtied + * afterward carry a higher generation and never delay this call. This is what + * lets us avoid io_wait_all, which waits for *global* I/O quiescence -- a + * condition a busy cache (background cleaner writes, async reads) may never + * reach. + * + * A single scan of the entries suffices, because a generation only ever moves + * above the cut (a page must go clean before it can be re-dirtied), so an entry + * that has left the pre-cut set never re-enters it. We first issue writeback + * for every dirty, unlocked page in one bulk pass so the writes pipeline, then + * walk the entries once, waiting per entry for its pre-cut interval to drain. + * + * The bulk pass leaves every pre-cut page that belongs to this checkpoint in + * CC_WRITEBACK: such a page is dirty and, under copy-on-write, never locked or + * claimed, so clockcache_try_set_writeback issues it (or a concurrent cleaner + * already has). The drain therefore only has to wait on pages that are in + * writeback; a pre-cut page that is not in writeback is either already clean, + * or was held by a writer during the bulk pass and so cannot belong to this + * checkpoint -- either way we skip it, which is what keeps this deadlock-free. + * Per-context background reapers complete the issued writes; we also poll our + * own context via clockcache_wait to help things along. + * + * Termination: once a page is in writeback the writer is excluded (see + * clockcache_get_write), so it progresses to clean and, if re-dirtied, moves to + * a generation above the cut. + *-------------------------------------------------------------------------- + */ +platform_status +clockcache_writeback_dirty(clockcache *cc) +{ + uint64 cutoff = + __atomic_fetch_add(&cc->dirty_generation, 1, __ATOMIC_SEQ_CST); + platform_assert(cutoff < UINT64_MAX); + + // Bulk-issue writeback for every dirty, unlocked page so the writes pipeline + // rather than draining one batch at a time. + for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { + platform_status result = + clockcache_batch_start_writeback(cc, batch, TRUE); + if (!SUCCESS(result)) { + return result; + } + } + + // Wait for each pre-cut interval's writeback to complete, in a single pass. + for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_no); + while (TRUE) { + uint64 gen = + __atomic_load_n(&entry->dirty_generation, __ATOMIC_RELAXED); + if (gen == 0 || gen > cutoff + || !clockcache_test_flag(cc, entry_no, CC_WRITEBACK)) + { + break; + } + clockcache_wait(cc); + } + } + + return STATUS_OK; } /* @@ -1147,8 +1290,9 @@ clockcache_try_evict(clockcache *cc, uint32 entry_number) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); - entry->type = PAGE_TYPE_INVALID; - entry->status = CC_FREE_STATUS; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; + entry->status = CC_FREE_STATUS; clockcache_log( addr, entry_number, "evict: entry %u addr %lu\n", entry_number, addr); @@ -1228,7 +1372,12 @@ clockcache_move_hand(clockcache *cc, bool32 is_urgent) cleaner_hand = (evict_hand + cc->cleaner_gap) % cc->cfg->batch_capacity; clean_batch_busy = &cc->batch_busy[cleaner_hand]; if (__sync_bool_compare_and_swap(clean_batch_busy, FALSE, TRUE)) { - clockcache_batch_start_writeback(cc, cleaner_hand, is_urgent); + platform_status rc = + clockcache_batch_start_writeback(cc, cleaner_hand, is_urgent); + if (!SUCCESS(rc)) { + platform_error_log("clockcache_move_hand: writeback failed: %s\n", + platform_status_to_string(rc)); + } was_busy = __sync_bool_compare_and_swap(clean_batch_busy, TRUE, FALSE); debug_assert(was_busy); } @@ -1279,12 +1428,22 @@ clockcache_get_free_page(clockcache *cc, && __sync_bool_compare_and_swap( &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS)) { + + // A page that begins dirty (a fresh allocation) must carry a dirty + // generation, just like a clean->dirty transition. The entry is + // write-locked here, so a concurrent fence skips it regardless. + if (!(status & CC_CLEAN)) { + debug_assert(entry->dirty_generation == 0); + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + } + entry->status = status; + entry->type = type; + if (refcount) { clockcache_inc_ref(cc, entry_no, tid); } platform_assert(entry->waiters.head == NULL); - entry->status = status; - entry->type = type; debug_assert(entry->page.disk_addr == CC_UNMAPPED_ADDR); clockcache_record_backtrace(cc, entry_no); return entry_no; @@ -1343,7 +1502,9 @@ clockcache_flush(clockcache *cc) flush_hand < cc->cfg->page_capacity / CC_ENTRIES_PER_BATCH; flush_hand++) { - clockcache_batch_start_writeback(cc, flush_hand, TRUE); + platform_status rc = + clockcache_batch_start_writeback(cc, flush_hand, TRUE); + platform_assert_status_ok(rc); } // make sure all aio is complete again @@ -1503,8 +1664,9 @@ clockcache_try_page_discard(clockcache *cc, uint64 addr) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); - entry->type = PAGE_TYPE_INVALID; - entry->status = CC_FREE_STATUS; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; + entry->status = CC_FREE_STATUS; /* 7. reset pincount */ clockcache_reset_pin(cc, entry_number); @@ -1621,8 +1783,9 @@ clockcache_get_in_cache(clockcache *cc, // IN static void clockcache_release_unpublished_entry(clockcache_entry *entry) { - entry->page.disk_addr = CC_UNMAPPED_ADDR; - entry->type = PAGE_TYPE_INVALID; + entry->page.disk_addr = CC_UNMAPPED_ADDR; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; platform_assert(entry->waiters.head == NULL); entry->status = CC_FREE_STATUS; } @@ -2166,6 +2329,10 @@ clockcache_lock(clockcache *cc, page_handle *page) entry_number, page->disk_addr); clockcache_get_write(cc, entry_number); + // A write lock marks the page dirty (and stamps its dirty generation). The + // CC_WRITEBACK exclusion in clockcache_get_write guarantees this cannot race + // a writeback completion. + clockcache_dirty_begin(cc, entry_number); } void @@ -2185,27 +2352,6 @@ clockcache_unlock(clockcache *cc, page_handle *page) } -/*---------------------------------------------------------------------- - * clockcache_mark_dirty -- - * - * Marks the entry dirty. - *---------------------------------------------------------------------- - */ -void -clockcache_mark_dirty(clockcache *cc, page_handle *page) -{ - debug_only clockcache_entry *entry = clockcache_page_to_entry(cc, page); - uint32 entry_number = clockcache_page_to_entry_number(cc, page); - - clockcache_log(entry->page.disk_addr, - entry_number, - "mark_dirty: entry %u addr %lu\n", - entry_number, - entry->page.disk_addr); - clockcache_clear_flag(cc, entry_number, CC_CLEAN); - return; -} - /* *---------------------------------------------------------------------- * clockcache_pin -- @@ -2248,17 +2394,22 @@ clockcache_unpin(clockcache *cc, page_handle *page) /* *----------------------------------------------------------------------------- - * clockcache_page_sync -- + * clockcache_page_writeback -- * - * Asynchronously syncs the page. Currently there is no way to check - *when the writeback has completed. + * Issues writeback of the page. This does not make the page durable; it + * only hands the write to the I/O layer. + * + * With is_blocking == FALSE the writeback is issued asynchronously and + * this returns without waiting for completion. With is_blocking == TRUE + * the page is written synchronously and has completed by the time this + * returns. *----------------------------------------------------------------------------- */ void -clockcache_page_sync(clockcache *cc, - page_handle *page, - bool32 is_blocking, - page_type type) +clockcache_page_writeback(clockcache *cc, + page_handle *page, + bool32 is_blocking, + page_type type) { uint32 entry_number = clockcache_page_to_entry_number(cc, page); async_io_state *state; @@ -2266,9 +2417,26 @@ clockcache_page_sync(clockcache *cc, const threadid tid = platform_get_tid(); platform_status status; - if (!clockcache_try_set_writeback(cc, entry_number, TRUE)) { - platform_assert(clockcache_test_flag(cc, entry_number, CC_CLEAN)); - return; + while (!clockcache_try_set_writeback(cc, entry_number, TRUE)) { + if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { + return; + } + + /* + * A pressure cleaner or a checkpoint fence may have begun writeback + * after the caller released its claim. Wait for that writeback rather + * than treating a perfectly valid concurrent flush as an assertion. + */ + if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK)) { + clockcache_wait(cc); + continue; + } + + platform_assert(0, + "page_writeback requires a cleanable page: entry=%u " + "status=%u\n", + entry_number, + clockcache_get_status(cc, entry_number)); } if (cc->cfg->use_stats) { @@ -2279,11 +2447,12 @@ clockcache_page_sync(clockcache *cc, if (!is_blocking) { state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); if (state == NULL) { - platform_error_log("clockcache_page_sync: async_io_state allocation " - "failed for addr %lu, entry %u, type %u\n", - addr, - entry_number, - type); + platform_error_log( + "clockcache_page_writeback: async_io_state allocation " + "failed for addr %lu, entry %u, type %u\n", + addr, + entry_number, + type); } platform_assert(state); state->cc = cc; @@ -2295,65 +2464,68 @@ clockcache_page_sync(clockcache *cc, clockcache_write_callback, state); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_async_state_init failed " - "for addr %lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_async_state_init failed " + "for addr %lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); status = io_async_state_append_page(state->iostate, page->data); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_async_state_append_page " - "failed for addr %lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_async_state_append_page " + "failed for addr %lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); io_async_run(state->iostate); } else { status = io_write(cc->io, page->data, clockcache_page_size(cc), addr); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_write failed for addr " - "%lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_write failed for addr " + "%lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); clockcache_log(addr, entry_number, - "page_sync write entry %u addr %lu\n", + "page_writeback write entry %u addr %lu\n", entry_number, addr); - debug_only uint8 rc; - rc = clockcache_set_flag(cc, entry_number, CC_CLEAN); - debug_assert(!rc); - rc = clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); - debug_assert(rc); + clockcache_dirty_complete_writeback(cc, entry_number); } } /* *----------------------------------------------------------------------------- - * clockcache_extent_sync -- + * clockcache_extent_writeback -- * - * Asynchronously syncs the extent. + * Asynchronously issues writeback of the extent. This does not make the + * extent durable; it only hands the writes to the I/O layer and returns + * without waiting for completion. * * Adds the number of pages issued writeback to the counter pointed to * by pages_outstanding. When the writes complete, a callback subtracts * them off, so that the caller may track how many pages are in - *writeback. + * writeback. * * Assumes all pages in the extent are clean or cleanable *----------------------------------------------------------------------------- */ void -clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) +clockcache_extent_writeback(clockcache *cc, + uint64 addr, + uint64 *pages_outstanding) { async_io_state *state = NULL; uint64 i; @@ -2372,7 +2544,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) req_addr = page_addr; state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); if (state == NULL) { - platform_error_log("clockcache_extent_sync: async_io_state " + platform_error_log("clockcache_extent_writeback: async_io_state " "allocation failed for extent addr %lu, " "page addr %lu, entry %u\n", addr, @@ -2389,7 +2561,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) clockcache_write_callback, state); if (!SUCCESS(rc)) { - platform_error_log("clockcache_extent_sync: " + platform_error_log("clockcache_extent_writeback: " "io_async_state_init failed for extent addr " "%lu, req addr %lu, entry %u: %s\n", addr, @@ -2402,7 +2574,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) platform_status rc = io_async_state_append_page( state->iostate, clockcache_get_entry(cc, entry_number)->page.data); if (!SUCCESS(rc)) { - platform_error_log("clockcache_extent_sync: " + platform_error_log("clockcache_extent_writeback: " "io_async_state_append_page failed for extent " "addr %lu, page addr %lu, entry %u: %s\n", addr, @@ -3095,13 +3267,6 @@ clockcache_prefetch_page_virtual(cache *c, uint64 addr, page_type type) clockcache_prefetch_page(cc, addr, type); } -void -clockcache_mark_dirty_virtual(cache *c, page_handle *page) -{ - clockcache *cc = (clockcache *)c; - clockcache_mark_dirty(cc, page); -} - void clockcache_pin_virtual(cache *c, page_handle *page) { @@ -3148,20 +3313,22 @@ clockcache_get_async_state_result_virtual(void *payload) } void -clockcache_page_sync_virtual(cache *c, - page_handle *page, - bool32 is_blocking, - page_type type) +clockcache_page_writeback_virtual(cache *c, + page_handle *page, + bool32 is_blocking, + page_type type) { clockcache *cc = (clockcache *)c; - clockcache_page_sync(cc, page, is_blocking, type); + clockcache_page_writeback(cc, page, is_blocking, type); } void -clockcache_extent_sync_virtual(cache *c, uint64 addr, uint64 *pages_outstanding) +clockcache_extent_writeback_virtual(cache *c, + uint64 addr, + uint64 *pages_outstanding) { clockcache *cc = (clockcache *)c; - clockcache_extent_sync(cc, addr, pages_outstanding); + clockcache_extent_writeback(cc, addr, pages_outstanding); } void @@ -3171,6 +3338,20 @@ clockcache_flush_virtual(cache *c) clockcache_flush(cc); } +platform_status +clockcache_writeback_dirty_virtual(cache *c) +{ + clockcache *cc = (clockcache *)c; + return clockcache_writeback_dirty(cc); +} + +platform_status +clockcache_durable_barrier_virtual(cache *c) +{ + clockcache *cc = (clockcache *)c; + return io_durable_barrier(cc->io); +} + int clockcache_evict_all_virtual(cache *c, bool32 ignore_pinned) { @@ -3299,12 +3480,13 @@ static cache_ops clockcache_ops = { .page_unlock = clockcache_unlock_virtual, .page_prefetch = clockcache_prefetch_virtual, .page_prefetch_page = clockcache_prefetch_page_virtual, - .page_mark_dirty = clockcache_mark_dirty_virtual, .page_pin = clockcache_pin_virtual, .page_unpin = clockcache_unpin_virtual, - .page_sync = clockcache_page_sync_virtual, - .extent_sync = clockcache_extent_sync_virtual, + .page_writeback = clockcache_page_writeback_virtual, + .extent_writeback = clockcache_extent_writeback_virtual, .flush = clockcache_flush_virtual, + .writeback_dirty = clockcache_writeback_dirty_virtual, + .durable_barrier = clockcache_durable_barrier_virtual, .evict = clockcache_evict_all_virtual, .cleanup = clockcache_wait_virtual, .in_use = clockcache_in_use_virtual, @@ -3397,6 +3579,10 @@ clockcache_init(clockcache *cc, // OUT cc->io = io; cc->heap_id = hid; + // Generation 0 is the sentinel for "clean/free", so dirty stamping starts + // at 1. + cc->dirty_generation = 1; + /* lookup maps addrs to entries, entry contains the entries themselves */ platform_status rc = platform_buffer_init( &cc->lookup_bh, allocator_page_capacity * sizeof(cc->lookup[0])); @@ -3438,9 +3624,10 @@ clockcache_init(clockcache *cc, // OUT for (i = 0; i < cc->cfg->page_capacity; i++) { cc->entry[i].page.data = cc->data + clockcache_multiply_by_page_size(cc, i); - cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; - cc->entry[i].status = CC_FREE_STATUS; - cc->entry[i].type = PAGE_TYPE_INVALID; + cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; + cc->entry[i].status = CC_FREE_STATUS; + cc->entry[i].dirty_generation = 0; + cc->entry[i].type = PAGE_TYPE_INVALID; async_wait_queue_init(&cc->entry[i].waiters); } diff --git a/src/clockcache.h b/src/clockcache.h index 814cde6a..cb5929fd 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -72,8 +72,13 @@ typedef uint32 entry_status; // Saved in clockcache_entry->status struct clockcache_entry { page_handle page; volatile entry_status status; - page_type type; - async_wait_queue waiters; + // Generation in which this page's current dirty interval began; 0 when the + // page is clean or free. A writeback_dirty() drains every entry whose + // generation is at or below the cut it took (see + // clockcache_writeback_dirty). + volatile uint64 dirty_generation; + page_type type; + async_wait_queue waiters; #ifdef RECORD_ACQUISITION_STACKS int next_history_record; history_record history[NUM_HISTORY_RECORDS]; @@ -139,6 +144,11 @@ struct clockcache { volatile bool32 *batch_busy; // Convenience pointer for batch_bh uint64 cleaner_gap; + // Monotonic generation counter. A writeback_dirty() atomically increments it + // to take a "cut", then drains every entry stamped with a generation at or + // below that cut. + uint64 dirty_generation; + volatile struct { volatile uint32 free_hand; bool32 enable_sync_get; diff --git a/src/core.c b/src/core.c index d499f24a..d17af7bc 100644 --- a/src/core.c +++ b/src/core.c @@ -46,8 +46,37 @@ static const int64 latency_histo_buckets[LATENCYHISTO_SIZE] = { _Static_assert(CORE_NUM_MEMTABLES <= MAX_MEMTABLES, "CORE_NUM_MEMTABLES <= MAX_MEMTABLES"); -/* Some randomly chosen Splinter super-block checksum seed. */ -#define CORE_SUPER_CSUM_SEED (42) +/* Checkpoint metadata has independently checksummed directory and records. */ +#define CORE_CHECKPOINT_DIRECTORY_CSUM_SEED (42) +#define CORE_CHECKPOINT_RECORD_CSUM_SEED (43) + +#define CORE_CHECKPOINT_FORMAT_VERSION (2) +#define CORE_CHECKPOINT_RECORD_COUNT (2) + +#define CORE_CHECKPOINT_DIRECTORY_MAGIC (0x534442434B505444ULL) // SDBCKPTD +#define CORE_CHECKPOINT_RECORD_MAGIC (0x534442434B505452ULL) // SDBCKPTR + +static platform_status +core_checkpoint_lock_init(core_handle *spl) +{ + platform_status rc = platform_mutex_init( + &spl->checkpoint_lock, platform_get_module_id(), spl->heap_id); + if (SUCCESS(rc)) { + spl->checkpoint_lock_initialized = TRUE; + } + return rc; +} + +static void +core_checkpoint_lock_deinit(core_handle *spl) +{ + if (!spl->checkpoint_lock_initialized) { + return; + } + platform_status rc = platform_mutex_destroy(&spl->checkpoint_lock); + platform_assert_status_ok(rc); + spl->checkpoint_lock_initialized = FALSE; +} /* * core logging functions. @@ -107,142 +136,530 @@ core_close_log_stream_if_enabled(core_handle *spl, /* *----------------------------------------------------------------------------- - * Splinter Super Block: Disk-resident structure. - * Super block lives on page of page type == PAGE_TYPE_SUPERBLOCK. + * Checkpoint metadata: disk-resident structures. + * + * allocator_get_super_addr() identifies a fixed page in the allocator's + * bootstrap extent. That page is an immutable directory, written exactly + * once when the table is created. The directory names two independently + * allocated record extents. Checkpoint publication alternates between their + * first pages, leaving one formerly valid record untouched if a new write is + * torn. + * + * This is intentionally a new on-disk format. Do not interpret a legacy + * core_super_block as a directory: doing so would turn arbitrary old fields + * into allocator-owned addresses. Existing databases must be migrated or + * reformatted before using this checkpoint metadata format. *----------------------------------------------------------------------------- */ -typedef struct ONDISK core_super_block { - uint64 root_addr; // Address of the root of the trunk for the instance - // referenced by this superblock. - uint64 log_addr; - uint64 log_meta_addr; +typedef struct ONDISK core_checkpoint_directory { + uint64 magic; + uint64 format_version; + uint64 table_id; + uint64 record_addr[CORE_CHECKPOINT_RECORD_COUNT]; + checksum128 checksum; +} core_checkpoint_directory; + +typedef struct ONDISK core_checkpoint_record { + /* + * The highest memtable generation incorporated in root_addr. The boolean + * keeps the fresh-database case distinct from generation zero. + */ + uint64 incorporated_generation; + bool32 has_incorporated_generation; + uint64 root_addr; uint64 timestamp; + uint64 sequence; + uint64 table_id; + uint32 record_slot; bool32 checkpointed; bool32 unmounted; + uint64 magic; + uint64 format_version; checksum128 checksum; -} core_super_block; +} core_checkpoint_record; + +typedef struct core_checkpoint_records { + core_checkpoint_record record[CORE_CHECKPOINT_RECORD_COUNT]; + bool32 valid[CORE_CHECKPOINT_RECORD_COUNT]; + bool32 have_newest; + uint64 newest_slot; + bool32 have_newest_unmounted; + uint64 newest_unmounted_slot; +} core_checkpoint_records; + +static checksum128 +core_checkpoint_directory_checksum(const core_checkpoint_directory *directory) +{ + return platform_checksum128(directory, + offsetof(core_checkpoint_directory, checksum), + CORE_CHECKPOINT_DIRECTORY_CSUM_SEED); +} + +static checksum128 +core_checkpoint_record_checksum(const core_checkpoint_record *record) +{ + return platform_checksum128(record, + offsetof(core_checkpoint_record, checksum), + CORE_CHECKPOINT_RECORD_CSUM_SEED); +} + +static bool32 +core_checkpoint_record_addr_is_valid(core_handle *spl, uint64 addr) +{ + allocator_config *allocator_cfg = allocator_get_config(spl->al); + uint64 page_size = cache_page_size(spl->cc); + + return addr != 0 && addr % allocator_cfg->io_cfg->extent_size == 0 + && addr < allocator_cfg->capacity + && page_size <= allocator_cfg->capacity - addr; +} + +static bool32 +core_checkpoint_directory_is_valid(core_handle *spl, + const core_checkpoint_directory *directory) +{ + if (directory->magic != CORE_CHECKPOINT_DIRECTORY_MAGIC + || directory->format_version != CORE_CHECKPOINT_FORMAT_VERSION + || directory->table_id != spl->id + || !platform_checksum_is_equal( + directory->checksum, core_checkpoint_directory_checksum(directory))) + { + return FALSE; + } + + uint64 record0 = directory->record_addr[0]; + uint64 record1 = directory->record_addr[1]; + allocator_config *allocator_cfg = allocator_get_config(spl->al); + return core_checkpoint_record_addr_is_valid(spl, record0) + && core_checkpoint_record_addr_is_valid(spl, record1) + && record0 != record1 + && !allocator_config_pages_share_extent( + allocator_cfg, record0, record1); +} + +static bool32 +core_checkpoint_record_is_valid(core_handle *spl, + const core_checkpoint_record *record, + uint64 record_slot) +{ + return record->magic == CORE_CHECKPOINT_RECORD_MAGIC + && record->format_version == CORE_CHECKPOINT_FORMAT_VERSION + && record->table_id == spl->id && record->record_slot == record_slot + && record->sequence != 0 + && (record->has_incorporated_generation == FALSE + || record->has_incorporated_generation == TRUE) + && (record->has_incorporated_generation + ? record->incorporated_generation < UINT64_MAX + : record->incorporated_generation == 0) + && platform_checksum_is_equal( + record->checksum, core_checkpoint_record_checksum(record)); +} + +static void +core_write_checkpoint_page(core_handle *spl, + uint64 page_addr, + const void *contents, + uint64 contents_size) +{ + page_handle *page = + cache_get(spl->cc, page_addr, TRUE, PAGE_TYPE_SUPERBLOCK); + uint64 wait = 1; + while (!cache_try_claim(spl->cc, page)) { + cache_unget(spl->cc, page); + platform_sleep_ns(wait); + wait = wait > 1024 ? wait : 2 * wait; + page = cache_get(spl->cc, page_addr, TRUE, PAGE_TYPE_SUPERBLOCK); + } + cache_lock(spl->cc, page); + platform_assert(contents_size <= cache_page_size(spl->cc)); + memset(page->data, 0, cache_page_size(spl->cc)); + memcpy(page->data, contents, contents_size); + cache_unlock(spl->cc, page); + cache_unclaim(spl->cc, page); + cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); + cache_unget(spl->cc, page); +} + +static void +core_initialize_checkpoint_record_page(core_handle *spl, uint64 page_addr) +{ + page_handle *page = cache_alloc(spl->cc, page_addr, PAGE_TYPE_SUPERBLOCK); + platform_assert(page != NULL); + memset(page->data, 0, cache_page_size(spl->cc)); + cache_unlock(spl->cc, page); + cache_unclaim(spl->cc, page); + cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); + cache_unget(spl->cc, page); +} + +static platform_status +core_create_checkpoint_directory(core_handle *spl, + core_checkpoint_directory *directory) +{ + uint64 directory_addr; + platform_status rc = + allocator_alloc_super_addr(spl->al, spl->id, &directory_addr); + if (!SUCCESS(rc)) { + platform_error_log("core_create_checkpoint_directory: failed to allocate " + "directory address for root id %lu: %s\n", + spl->id, + platform_status_to_string(rc)); + return rc; + } + + ZERO_CONTENTS(directory); + directory->magic = CORE_CHECKPOINT_DIRECTORY_MAGIC; + directory->format_version = CORE_CHECKPOINT_FORMAT_VERSION; + directory->table_id = spl->id; + + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + uint64 record_addr; + rc = allocator_alloc(spl->al, &record_addr, PAGE_TYPE_SUPERBLOCK); + if (!SUCCESS(rc)) { + platform_error_log("core_create_checkpoint_directory: failed to " + "allocate record extent %lu: %s\n", + slot, + platform_status_to_string(rc)); + return rc; + } + directory->record_addr[slot] = record_addr; + core_initialize_checkpoint_record_page(spl, record_addr); + } + + directory->checksum = core_checkpoint_directory_checksum(directory); + core_write_checkpoint_page( + spl, directory_addr, directory, sizeof(*directory)); + + /* + * Submit the newly initialized record and directory pages before the + * durable barrier. allocator_alloc() changes its refcount map only in + * memory; crash recovery deliberately rebuilds that map instead of relying + * on this publication. allocator_alloc_super_addr() does write the raw + * bootstrap mapping through the shared backing I/O handle, which the + * following durable barrier fdatasyncs with the directory page. + */ + rc = cache_writeback_dirty(spl->cc); + if (!SUCCESS(rc)) { + return rc; + } + return cache_durable_barrier(spl->cc); +} + +static platform_status +core_get_checkpoint_directory(core_handle *spl, + core_checkpoint_directory *directory) +{ + uint64 directory_addr; + platform_status rc = + allocator_get_super_addr(spl->al, spl->id, &directory_addr); + if (!SUCCESS(rc)) { + return rc; + } + + page_handle *page = + cache_get(spl->cc, directory_addr, TRUE, PAGE_TYPE_SUPERBLOCK); + memcpy(directory, page->data, sizeof(*directory)); + cache_unget(spl->cc, page); + + if (!core_checkpoint_directory_is_valid(spl, directory)) { + platform_error_log("core_get_checkpoint_directory: no compatible " + "checkpoint directory for root id %lu\n", + spl->id); + return STATUS_BAD_PARAM; + } + return STATUS_OK; +} + +static platform_status +core_load_checkpoint_records(core_handle *spl, + const core_checkpoint_directory *directory, + core_checkpoint_records *records) +{ + ZERO_CONTENTS(records); + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + page_handle *page = cache_get( + spl->cc, directory->record_addr[slot], TRUE, PAGE_TYPE_SUPERBLOCK); + memcpy(&records->record[slot], page->data, sizeof(records->record[slot])); + cache_unget(spl->cc, page); + + records->valid[slot] = + core_checkpoint_record_is_valid(spl, &records->record[slot], slot); + if (!records->valid[slot]) { + continue; + } + + if (!records->have_newest + || records->record[records->newest_slot].sequence + < records->record[slot].sequence) + { + records->have_newest = TRUE; + records->newest_slot = slot; + } + if (records->record[slot].unmounted + && (!records->have_newest_unmounted + || records->record[records->newest_unmounted_slot].sequence + < records->record[slot].sequence)) + { + records->have_newest_unmounted = TRUE; + records->newest_unmounted_slot = slot; + } + } + + if (records->valid[0] && records->valid[1] + && records->record[0].sequence == records->record[1].sequence) + { + platform_error_log("core_load_checkpoint_records: duplicate record " + "sequence %lu for root id %lu\n", + records->record[0].sequence, + spl->id); + return STATUS_BAD_PARAM; + } + return STATUS_OK; +} + +static void +core_destroy_checkpoint_record_extent(core_handle *spl, uint64 record_addr) +{ + refcount ref = allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); + if (ref != AL_NO_REFS) { + platform_error_log("core_destroy_checkpoint_record_extent: record extent " + "%lu has unexpected refcount %u\n", + record_addr, + ref); + return; + } + + cache_extent_discard(spl->cc, record_addr, PAGE_TYPE_SUPERBLOCK); + ref = allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); + platform_assert(ref == AL_FREE); +} + +static void +core_destroy_checkpoint_storage(core_handle *spl) +{ + core_checkpoint_directory directory; + platform_status rc = core_get_checkpoint_directory(spl, &directory); + if (!SUCCESS(rc)) { + platform_error_log("core_destroy_checkpoint_storage: unable to load " + "checkpoint directory for root id %lu: %s\n", + spl->id, + platform_status_to_string(rc)); + return; + } + + core_checkpoint_records records; + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + platform_error_log("core_destroy_checkpoint_storage: unable to load " + "checkpoint records for root id %lu: %s\n", + spl->id, + platform_status_to_string(rc)); + return; + } + + /* + * Both valid slots own independent root references. Keeping the older + * record live makes it a real fallback if the next record write is torn; + * its reference is released only when that slot is successfully + * overwritten. This is clean destruction, so release both record owners. + */ + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + if (!records.valid[slot] || records.record[slot].root_addr == 0) { + continue; + } + trunk_snapshot old_snapshot = {.root_addr = records.record[slot].root_addr}; + rc = trunk_snapshot_release(&spl->trunk_context, &old_snapshot); + if (!SUCCESS(rc)) { + platform_error_log("core_destroy_checkpoint_storage: failed to " + "release record %lu root %lu: %s\n", + slot, + records.record[slot].root_addr, + platform_status_to_string(rc)); + } + } + + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + core_destroy_checkpoint_record_extent(spl, directory.record_addr[slot]); + } +} /* *----------------------------------------------------------------------------- - * Super block functions + * Checkpoint record functions *----------------------------------------------------------------------------- */ static platform_status -core_set_super_block(core_handle *spl, - bool32 is_checkpoint, - bool32 is_unmount, - bool32 is_create) +core_capture_checkpoint_cut(core_handle *spl, + trunk_snapshot *snapshot, + bool32 *has_incorporated_generation, + uint64 *incorporated_generation) { - uint64 super_addr; - page_handle *super_page; - core_super_block *super; - uint64 wait = 1; - platform_status rc; + /* + * Incorporation publishes its generation and root while holding lookup + * exclusion before taking the trunk root lock. Take the checkpoint cut + * in the same order, so a record never combines a pre-incorporation + * generation with a post-incorporation root (or the converse). + */ + memtable_block_lookups(&spl->mt_ctxt); + uint64 retired_generation = memtable_generation_retired(&spl->mt_ctxt); + platform_status rc = trunk_snapshot_create(&spl->trunk_context, snapshot); + memtable_unblock_lookups(&spl->mt_ctxt); + if (!SUCCESS(rc)) { + return rc; + } + + *has_incorporated_generation = retired_generation != UINT64_MAX; + *incorporated_generation = + *has_incorporated_generation ? retired_generation : 0; + return STATUS_OK; +} + +static platform_status +core_publish_checkpoint_record(core_handle *spl, + bool32 is_checkpoint, + bool32 is_unmount, + bool32 is_create) +{ + uint64 old_root_addr; + platform_status rc; + trunk_snapshot snapshot; + bool32 has_incorporated_generation; + uint64 incorporated_generation; + core_checkpoint_directory directory; + core_checkpoint_records records; + uint64 target_slot; + core_checkpoint_record record; + + /* + * The snapshot, target-slot selection, durable record write, and old-slot + * release are one publication transaction. In particular, two concurrent + * publishers must never choose the same target slot or release the same + * former record owner. + */ + rc = platform_mutex_lock(&spl->checkpoint_lock); + if (!SUCCESS(rc)) { + return rc; + } + + rc = core_capture_checkpoint_cut( + spl, &snapshot, &has_incorporated_generation, &incorporated_generation); + if (!SUCCESS(rc)) { + goto unlock_checkpoint; + } + + /* + * The snapshot reference makes the root stable, but not necessarily + * durable. Drain only the cache intervals that existed at this cut before + * making a record that can name the root durable. This can incidentally + * persist newer log/data pages, but it does not seal or publish a logical + * durable-log tail; tail sync is a separate operation. + */ + rc = cache_writeback_dirty(spl->cc); + if (!SUCCESS(rc)) { + goto release_snapshot; + } + rc = cache_durable_barrier(spl->cc); + if (!SUCCESS(rc)) { + goto release_snapshot; + } if (is_create) { - rc = allocator_alloc_super_addr(spl->al, spl->id, &super_addr); + rc = core_create_checkpoint_directory(spl, &directory); } else { - rc = allocator_get_super_addr(spl->al, spl->id, &super_addr); + rc = core_get_checkpoint_directory(spl, &directory); } if (!SUCCESS(rc)) { - platform_error_log("core_set_super_block: failed to %s super block " - "address for root id %lu: %s\n", - is_create ? "allocate" : "get", + platform_error_log("core_publish_checkpoint_record: failed to %s " + "checkpoint directory for root id %lu: %s\n", + is_create ? "create" : "load", spl->id, platform_status_to_string(rc)); - return rc; - } - super_page = cache_get(spl->cc, super_addr, TRUE, PAGE_TYPE_SUPERBLOCK); - while (!cache_try_claim(spl->cc, super_page)) { - platform_sleep_ns(wait); - wait *= 2; + goto release_snapshot; } - wait = 1; - cache_lock(spl->cc, super_page); - super = (core_super_block *)super_page->data; - uint64 old_root_addr = super->root_addr; + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + goto release_snapshot; + } - trunk_ondisk_node_handle root_handle; - trunk_init_root_handle(&spl->trunk_context, &root_handle); - uint64 root_addr = trunk_ondisk_node_handle_addr(&root_handle); - if (root_addr != 0) { - trunk_inc_ref(spl->al, root_addr); + if (records.have_newest + && records.record[records.newest_slot].sequence == UINT64_MAX) + { + rc = STATUS_LIMIT_EXCEEDED; + goto release_snapshot; } - super->root_addr = root_addr; - trunk_ondisk_node_handle_deinit(&root_handle); + target_slot = records.have_newest ? records.newest_slot ^ 1 : 0; + /* + * Each valid record owns its root reference. This publication overwrites + * target_slot, so retain that slot's old root until the replacement page + * is durable, then release only the overwritten owner. The newest record + * remains independently live as the torn-write fallback. + */ + old_root_addr = + records.valid[target_slot] ? records.record[target_slot].root_addr : 0; + + ZERO_CONTENTS(&record); + record.incorporated_generation = incorporated_generation; + record.has_incorporated_generation = has_incorporated_generation; + record.root_addr = snapshot.root_addr; + record.timestamp = platform_get_real_time(); + record.sequence = records.have_newest + ? records.record[records.newest_slot].sequence + 1 + : 1; + record.table_id = spl->id; + record.record_slot = target_slot; + record.checkpointed = is_checkpoint; + record.unmounted = is_unmount; + record.magic = CORE_CHECKPOINT_RECORD_MAGIC; + record.format_version = CORE_CHECKPOINT_FORMAT_VERSION; + + record.checksum = core_checkpoint_record_checksum(&record); + + core_write_checkpoint_page( + spl, directory.record_addr[target_slot], &record, sizeof(record)); + /* The record now owns this reference, even if the barrier reports failure. + */ + snapshot.root_addr = 0; - if (spl->cfg.use_log) { - if (spl->log) { - super->log_addr = log_addr(spl->log); - super->log_meta_addr = log_meta_addr(spl->log); - } else { - super->log_addr = 0; - super->log_meta_addr = 0; - } + rc = cache_durable_barrier(spl->cc); + if (!SUCCESS(rc)) { + /* The new record may be durable, so retain its transferred root ref. */ + goto unlock_checkpoint; } - super->timestamp = platform_get_real_time(); - super->checkpointed = is_checkpoint; - super->unmounted = is_unmount; - super->checksum = - platform_checksum128(super, - sizeof(core_super_block) - sizeof(checksum128), - CORE_SUPER_CSUM_SEED); - - cache_mark_dirty(spl->cc, super_page); - cache_unlock(spl->cc, super_page); - cache_unclaim(spl->cc, super_page); - cache_unget(spl->cc, super_page); - cache_page_sync(spl->cc, super_page, TRUE, PAGE_TYPE_SUPERBLOCK); - - if (old_root_addr != 0 && !is_create) { - rc = trunk_dec_ref(spl->cfg.trunk_node_cfg, - PROCESS_PRIVATE_HEAP_ID, - spl->cc, - spl->al, - spl->ts, - old_root_addr); + + if (old_root_addr != 0) { + trunk_snapshot old_snapshot = {.root_addr = old_root_addr}; + rc = trunk_snapshot_release(&spl->trunk_context, &old_snapshot); if (!SUCCESS(rc)) { - platform_error_log("core_set_super_block: trunk_dec_ref failed for " - "old root addr %lu: %s\n", + platform_error_log("core_publish_checkpoint_record: " + "trunk_snapshot_release failed for old root addr " + "%lu: %s\n", old_root_addr, platform_status_to_string(rc)); - return rc; + goto unlock_checkpoint; } } - return STATUS_OK; -} + rc = STATUS_OK; + goto unlock_checkpoint; -static core_super_block * -core_get_super_block_if_valid(core_handle *spl, page_handle **super_page) +release_snapshot: { - uint64 super_addr; - core_super_block *super; - - platform_status rc = allocator_get_super_addr(spl->al, spl->id, &super_addr); - platform_assert_status_ok(rc); - *super_page = cache_get(spl->cc, super_addr, TRUE, PAGE_TYPE_SUPERBLOCK); - super = (core_super_block *)(*super_page)->data; - - if (!platform_checksum_is_equal( - super->checksum, - platform_checksum128(super, - sizeof(core_super_block) - sizeof(checksum128), - CORE_SUPER_CSUM_SEED))) - { - cache_unget(spl->cc, *super_page); - *super_page = NULL; - return NULL; + platform_status release_rc = + trunk_snapshot_release(&spl->trunk_context, &snapshot); + if (SUCCESS(rc) && !SUCCESS(release_rc)) { + rc = release_rc; } - - return super; } -static void -core_release_super_block(core_handle *spl, page_handle *super_page) +unlock_checkpoint: { - cache_unget(spl->cc, super_page); + platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); + if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { + rc = unlock_rc; + } +} + return rc; } /* @@ -418,6 +835,7 @@ core_begin_memtable_insert(core_handle *spl, uint64 *generation, memtable **mt) static platform_status core_log_insert(core_handle *spl, + uint64 memtable_generation, key tuple_key, message msg, const btree_insert_results *insert_results) @@ -436,8 +854,11 @@ core_log_insert(core_handle *spl, merge_accumulator_is_null(&insert_results->msg_blob) ? msg : merge_accumulator_to_message(&insert_results->msg_blob); - int log_rc = - log_write(spl->log, tuple_key, log_msg, insert_results->leaf_generation); + int log_rc = log_write(spl->log, + tuple_key, + log_msg, + memtable_generation, + insert_results->leaf_generation); return log_rc == 0 ? STATUS_OK : (platform_status){.r = log_rc}; } @@ -1554,7 +1975,7 @@ core_insert(core_handle *spl, goto end_insert; } - rc = core_log_insert(spl, tuple_key, data, &insert_results); + rc = core_log_insert(spl, generation, tuple_key, data, &insert_results); if (!SUCCESS(rc)) { goto end_insert; } @@ -1879,18 +2300,26 @@ core_mkfs(core_handle *spl, spl->heap_id = hid; spl->ts = ts; + platform_status rc = core_checkpoint_lock_init(spl); + if (!SUCCESS(rc)) { + platform_error_log( + "core_mkfs: checkpoint lock initialization failed: %s\n", + platform_status_to_string(rc)); + return rc; + } + // set up the memtable context memtable_config *mt_cfg = &spl->cfg.mt_cfg; - platform_status rc = memtable_context_init(&spl->mt_ctxt, - spl->heap_id, - cc, - mt_cfg, - core_memtable_flush_virtual, - spl); + rc = memtable_context_init(&spl->mt_ctxt, + spl->heap_id, + cc, + mt_cfg, + core_memtable_flush_virtual, + spl); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: memtable_context_init failed: %s\n", platform_status_to_string(rc)); - return rc; + goto deinit_checkpoint_lock; } // set up the log @@ -1903,8 +2332,13 @@ core_mkfs(core_handle *spl, } } - rc = trunk_context_init( - &spl->trunk_context, spl->cfg.trunk_node_cfg, hid, cc, al, ts, 0); + rc = trunk_context_init(&spl->trunk_context, + spl->cfg.trunk_node_cfg, + hid, + cc, + al, + ts, + (trunk_snapshot){.root_addr = 0}); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: trunk_context_init failed: %s\n", platform_status_to_string(rc)); @@ -1918,10 +2352,11 @@ core_mkfs(core_handle *spl, goto deinit_trunk_context; } - rc = core_set_super_block(spl, FALSE, FALSE, TRUE); + rc = core_publish_checkpoint_record(spl, FALSE, FALSE, TRUE); if (!SUCCESS(rc)) { - platform_error_log("core_mkfs: core_set_super_block failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mkfs: core_publish_checkpoint_record failed: %s\n", + platform_status_to_string(rc)); goto deinit_stats; } return STATUS_OK; @@ -1937,6 +2372,8 @@ core_mkfs(core_handle *spl, } deinit_memtable_context: memtable_context_deinit(&spl->mt_ctxt); +deinit_checkpoint_lock: + core_checkpoint_lock_deinit(spl); return rc; } @@ -1962,30 +2399,71 @@ core_mount(core_handle *spl, spl->heap_id = hid; spl->ts = ts; - // find the unmounted super block - uint64 root_addr = 0; - uint64 latest_timestamp = 0; - page_handle *super_page; - core_super_block *super = core_get_super_block_if_valid(spl, &super_page); - if (super != NULL) { - if (super->unmounted && super->timestamp > latest_timestamp) { - root_addr = super->root_addr; - latest_timestamp = super->timestamp; - } - core_release_super_block(spl, super_page); + platform_status rc = core_checkpoint_lock_init(spl); + if (!SUCCESS(rc)) { + platform_error_log( + "core_mount: checkpoint lock initialization failed: %s\n", + platform_status_to_string(rc)); + return rc; + } + + /* + * Preserve the historical clean-only mount rule for this first format + * slice: an interrupted run is not replayed yet, so only an explicitly + * unmounted record supplies the root. We still validate both records and + * choose the newest clean one by sequence rather than wall-clock time. + */ + uint64 root_addr = 0; + bool32 has_incorporated_generation = FALSE; + uint64 incorporated_generation = 0; + core_checkpoint_directory directory; + core_checkpoint_records records; + rc = core_get_checkpoint_directory(spl, &directory); + if (!SUCCESS(rc)) { + goto deinit_checkpoint_lock; } + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + goto deinit_checkpoint_lock; + } + if (!records.have_newest) { + platform_error_log("core_mount: checkpoint directory for root id %lu " + "has no valid records\n", + spl->id); + rc = STATUS_BAD_PARAM; + goto deinit_checkpoint_lock; + } + const core_checkpoint_record *record = &records.record[records.newest_slot]; + if (!record->unmounted) { + /* + * This is an interrupted run. An older clean record is only an A/B + * torn-write fallback, not permission to silently discard the newer + * checkpoint and its log suffix. Do not overwrite its metadata before + * log replay and allocator reconstruction are wired. + */ + platform_error_log("core_mount: root id %lu requires crash recovery\n", + spl->id); + rc = STATUS_INVALID_STATE; + goto deinit_checkpoint_lock; + } + root_addr = record->root_addr; + has_incorporated_generation = record->has_incorporated_generation; + incorporated_generation = record->incorporated_generation; memtable_config *mt_cfg = &spl->cfg.mt_cfg; - platform_status rc = memtable_context_init(&spl->mt_ctxt, - spl->heap_id, - cc, - mt_cfg, - core_memtable_flush_virtual, - spl); + rc = memtable_context_init_at_generation( + &spl->mt_ctxt, + spl->heap_id, + cc, + mt_cfg, + core_memtable_flush_virtual, + spl, + has_incorporated_generation ? incorporated_generation + 1 : 0); if (!SUCCESS(rc)) { - platform_error_log("core_mount: memtable_context_init failed: %s\n", + platform_error_log("core_mount: memtable_context_init_at_generation " + "failed: %s\n", platform_status_to_string(rc)); - return rc; + goto deinit_checkpoint_lock; } if (spl->cfg.use_log) { @@ -1997,8 +2475,22 @@ core_mount(core_handle *spl, } } - rc = trunk_context_init( - &spl->trunk_context, spl->cfg.trunk_node_cfg, hid, cc, al, ts, root_addr); + trunk_snapshot root_snapshot; + rc = trunk_snapshot_create_from_addr(al, root_addr, &root_snapshot); + if (!SUCCESS(rc)) { + platform_error_log( + "core_mount: trunk_snapshot_create_from_addr failed: %s\n", + platform_status_to_string(rc)); + goto deinit_log; + } + + rc = trunk_context_init(&spl->trunk_context, + spl->cfg.trunk_node_cfg, + hid, + cc, + al, + ts, + root_snapshot); if (!SUCCESS(rc)) { platform_error_log("core_mount: trunk_context_init failed: %s\n", platform_status_to_string(rc)); @@ -2012,10 +2504,11 @@ core_mount(core_handle *spl, goto deinit_trunk_context; } - rc = core_set_super_block(spl, FALSE, FALSE, FALSE); + rc = core_publish_checkpoint_record(spl, FALSE, FALSE, FALSE); if (!SUCCESS(rc)) { - platform_error_log("core_mount: core_set_super_block failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mount: core_publish_checkpoint_record failed: %s\n", + platform_status_to_string(rc)); goto deinit_stats; } return STATUS_OK; @@ -2031,6 +2524,8 @@ core_mount(core_handle *spl, } deinit_memtable_context: memtable_context_deinit(&spl->mt_ctxt); +deinit_checkpoint_lock: + core_checkpoint_lock_deinit(spl); return rc; } @@ -2082,9 +2577,11 @@ core_report_unincorporated_memtables(core_handle *spl) /* * This function is only safe to call when all other calls to spl have returned. + * It intentionally leaves the memtable and log contexts live: the clean + * checkpoint record needs both after final incorporation has quiesced. */ -void -core_prepare_for_shutdown(core_handle *spl) +static void +core_quiesce_for_shutdown(core_handle *spl) { // write current memtable to disk // (any others must already be flushing/flushed) @@ -2104,13 +2601,19 @@ core_prepare_for_shutdown(core_handle *spl) platform_assert_status_ok(rc); core_report_unincorporated_memtables(spl); +} - // destroy memtable context (and its memtables) +static void +core_teardown_after_shutdown(core_handle *spl) +{ + // Keep this after checkpoint publication: it supplies the generation cut. memtable_context_deinit(&spl->mt_ctxt); - // release the log + // Keep the log alive through clean-record publication. A later explicit + // tail-sync protocol will own its immutable log metadata separately. if (spl->cfg.use_log) { platform_free(spl->heap_id, spl->log); + spl->log = NULL; } // flush all dirty pages in the cache @@ -2126,14 +2629,22 @@ core_unmount(core_handle *spl) { platform_status rc; - core_prepare_for_shutdown(spl); - rc = core_set_super_block(spl, FALSE, TRUE, FALSE); + /* + * Quiescing leaves the memtable and log contexts live so publication can + * atomically capture the retired generation and root, then record log + * metadata. Teardown is safe regardless of publication success. + */ + core_quiesce_for_shutdown(spl); + rc = core_publish_checkpoint_record(spl, FALSE, TRUE, FALSE); if (!SUCCESS(rc)) { - platform_error_log("core_unmount: failed to update super block: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_unmount: failed to publish checkpoint record: %s\n", + platform_status_to_string(rc)); } + core_teardown_after_shutdown(spl); trunk_context_deinit(&spl->trunk_context); core_destroy_stats(spl); + core_checkpoint_lock_deinit(spl); return rc; } @@ -2143,12 +2654,16 @@ core_unmount(core_handle *spl) void core_destroy(core_handle *spl) { - core_prepare_for_shutdown(spl); + core_quiesce_for_shutdown(spl); + core_teardown_after_shutdown(spl); + /* Records own trunk references and their two dedicated record extents. */ + core_destroy_checkpoint_storage(spl); trunk_context_deinit(&spl->trunk_context); // clear out this splinter table from the meta page. allocator_remove_super_addr(spl->al, spl->id); core_destroy_stats(spl); + core_checkpoint_lock_deinit(spl); } @@ -2181,27 +2696,58 @@ core_print_space_use(platform_log_handle *log_handle, core_handle *spl) /* * core_print_super_block() * - * Fetch a super-block for a running Splinter instance, and print its - * contents. + * Print the fixed checkpoint directory and both independently written record + * slots for a running Splinter instance. */ void core_print_super_block(platform_log_handle *log_handle, core_handle *spl) { - page_handle *super_page; - core_super_block *super = core_get_super_block_if_valid(spl, &super_page); - if (super == NULL) { + core_checkpoint_directory directory; + platform_status rc = core_get_checkpoint_directory(spl, &directory); + if (!SUCCESS(rc)) { + platform_log(log_handle, + "No compatible checkpoint directory for root id %lu\n", + spl->id); + return; + } + + core_checkpoint_records records; + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + platform_log(log_handle, + "Unable to load checkpoint records for root id %lu: %s\n", + spl->id, + platform_status_to_string(rc)); return; } - platform_log(log_handle, "Superblock root_addr=%lu {\n", super->root_addr); - platform_log(log_handle, "log_meta_addr=%lu\n", super->log_meta_addr); platform_log(log_handle, - "timestamp=%lu, checkpointed=%d, unmounted=%d\n", - super->timestamp, - super->checkpointed, - super->unmounted); + "Checkpoint directory root_id=%lu record_addr=[%lu, %lu] {\n", + directory.table_id, + directory.record_addr[0], + directory.record_addr[1]); + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + if (!records.valid[slot]) { + platform_log(log_handle, " record[%lu]: invalid\n", slot); + continue; + } + core_checkpoint_record *record = &records.record[slot]; + platform_log(log_handle, + " record[%lu]: sequence=%lu root_addr=%lu " + "has_incorporated_generation=%d " + "incorporated_generation=%lu " + "timestamp=%lu " + "checkpointed=%d unmounted=%d\n", + slot, + record->sequence, + record->root_addr, + record->has_incorporated_generation, + record->incorporated_generation, + record->timestamp, + record->checkpointed, + record->unmounted); + } platform_log(log_handle, "}\n\n"); - core_release_super_block(spl, super_page); } // clang-format off diff --git a/src/core.h b/src/core.h index 94c10f97..997c0622 100644 --- a/src/core.h +++ b/src/core.h @@ -119,6 +119,10 @@ struct core_handle { trunk_context trunk_context; memtable_context mt_ctxt; + /* Serializes snapshot cuts and A/B checkpoint-record publication. */ + platform_mutex checkpoint_lock; + bool32 checkpoint_lock_initialized; + core_stats *stats; core_compacted_memtable compacted_memtable[MAX_MEMTABLES]; diff --git a/src/log.h b/src/log.h index 325d0764..30366edc 100644 --- a/src/log.h +++ b/src/log.h @@ -16,16 +16,55 @@ typedef struct log_handle log_handle; typedef struct log_iterator log_iterator; typedef struct log_config log_config; +/* + * Identity of one mini-allocator-backed log stream. It is sufficient for a + * higher-level checkpoint record to describe a stream, but not by itself a + * durable descriptor: core will later persist this information in an + * independently checksummed log-segment record. + */ +typedef struct log_segment_info { + uint64 addr; + uint64 meta_addr; + uint64 magic; +} log_segment_info; + typedef int (*log_write_fn)(log_handle *log, key tuple_key, message data, - uint64 generation); + uint64 memtable_generation, + uint64 leaf_generation); +/* + * Finalize the current append pages into checksummed, immutable pages. + * + * The caller must exclude concurrent log_write() calls until it has taken the + * cache writeback fence that is to make the pages durable. It must also + * serialize concurrent log_seal() calls. seal() itself does not issue I/O or + * a durable barrier. + * + * This is deliberately not a persisted replay-boundary descriptor. A log + * implementation whose metadata can grow after sealing needs an additional + * boundary in the checkpoint record to exclude those later entries. + */ +typedef platform_status (*log_seal_fn)(log_handle *log); +/* + * Detach the current stream after sealing it and prepare a distinct fresh + * stream. The caller must exclude writes throughout the operation and make + * the returned identities durable before allowing writes to the fresh stream. + * A zero sealed.meta_addr means the old stream was empty and discarded. + * Rotation alone does not advance the logical durable-log tail: that requires + * a separate, durable tail/manifest publication by the caller. + */ +typedef platform_status (*log_rotate_fn)(log_handle *log, + log_segment_info *sealed, + log_segment_info *fresh); typedef void (*log_release_fn)(log_handle *log); typedef uint64 (*log_addr_fn)(log_handle *log); typedef uint64 (*log_magic_fn)(log_handle *log); typedef struct log_ops { log_write_fn write; + log_seal_fn seal; + log_rotate_fn rotate; log_release_fn release; log_addr_fn addr; log_addr_fn meta_addr; @@ -38,9 +77,30 @@ struct log_handle { }; static inline int -log_write(log_handle *log, key tuple_key, message data, uint64 generation) +log_write(log_handle *log, + key tuple_key, + message data, + uint64 memtable_generation, + uint64 leaf_generation) +{ + return log->ops->write( + log, tuple_key, data, memtable_generation, leaf_generation); +} + +/* + * Finalize the log's current append pages. See log_seal_fn for the required + * exclusion, boundary, and durability ordering. + */ +static inline platform_status +log_seal(log_handle *log) +{ + return log->ops->seal(log); +} + +static inline platform_status +log_rotate(log_handle *log, log_segment_info *sealed, log_segment_info *fresh) { - return log->ops->write(log, tuple_key, data, generation); + return log->ops->rotate(log, sealed, fresh); } static inline void diff --git a/src/memtable.c b/src/memtable.c index 41482e68..e8b8969b 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -80,6 +80,20 @@ memtable_end_insert(memtable_context *ctxt) batch_rwlock_unget(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } +void +memtable_block_inserts(memtable_context *ctxt) +{ + batch_rwlock_get(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_claim_loop(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_lock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); +} + +void +memtable_unblock_inserts(memtable_context *ctxt) +{ + batch_rwlock_full_unlock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); +} + static inline bool32 memtable_try_begin_insert_rotation(memtable_context *ctxt) { @@ -97,20 +111,6 @@ memtable_end_insert_rotation(memtable_context *ctxt) batch_rwlock_unclaim(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } -static inline void -memtable_begin_raw_rotation(memtable_context *ctxt) -{ - batch_rwlock_get(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); - batch_rwlock_claim_loop(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); - batch_rwlock_lock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); -} - -static inline void -memtable_end_raw_rotation(memtable_context *ctxt) -{ - batch_rwlock_full_unlock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); -} - void memtable_begin_lookup(memtable_context *ctxt) { @@ -297,7 +297,7 @@ memtable_mark_incorporation_failed(memtable *mt, platform_status status) uint64 memtable_force_finalize(memtable_context *ctxt) { - memtable_begin_raw_rotation(ctxt); + memtable_block_inserts(ctxt); uint64 generation = ctxt->generation; uint64 mt_no = generation % ctxt->cfg.max_memtables; @@ -308,7 +308,7 @@ memtable_force_finalize(memtable_context *ctxt) <= ctxt->cfg.max_memtables); memtable_mark_empty(ctxt); - memtable_end_raw_rotation(ctxt); + memtable_unblock_inserts(ctxt); return current_generation; } @@ -357,12 +357,13 @@ memtable_deinit(cache *cc, memtable *mt) } platform_status -memtable_context_init(memtable_context *ctxt, - platform_heap_id hid, - cache *cc, - memtable_config *cfg, - process_fn process, - void *process_ctxt) +memtable_context_init_at_generation(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn process, + void *process_ctxt, + uint64 first_generation) { platform_status rc; ZERO_CONTENTS(ctxt); @@ -370,14 +371,22 @@ memtable_context_init(memtable_context *ctxt, ctxt->cfg = *cfg; ctxt->hid = hid; - if (MAX_MEMTABLES < cfg->max_memtables) { - platform_error_log("Configured number of memtables (%lu) exceeds max " - "supported memtables (%d)\n", + if (cfg->max_memtables == 0 || MAX_MEMTABLES < cfg->max_memtables) { + platform_error_log("Configured number of memtables (%lu) must be " + "between 1 and %d\n", cfg->max_memtables, MAX_MEMTABLES); return STATUS_BAD_PARAM; } + if (first_generation > (uint64)-1 - cfg->max_memtables) { + platform_error_log("First memtable generation (%lu) leaves too little " + "space for %lu memtable slots\n", + first_generation, + cfg->max_memtables); + return STATUS_BAD_PARAM; + } + rc = platform_mutex_init( &ctxt->incorporation_mutex, platform_get_module_id(), hid); if (!SUCCESS(rc)) { @@ -394,14 +403,24 @@ memtable_context_init(memtable_context *ctxt, batch_rwlock_init(&ctxt->rwlock); - for (uint64 mt_no = 0; mt_no < cfg->max_memtables; mt_no++) { - uint64 generation = mt_no; + for (uint64 generation_offset = 0; generation_offset < cfg->max_memtables; + generation_offset++) + { + uint64 generation = first_generation + generation_offset; + uint64 mt_no = generation % cfg->max_memtables; memtable_init(&ctxt->mt[mt_no], cc, cfg, generation); } - ctxt->generation = 0; - ctxt->generation_to_incorporate = 0; - ctxt->generation_retired = (uint64)-1; + ctxt->generation = first_generation; + ctxt->generation_to_incorporate = first_generation; + /* + * A fresh database has no incorporated generation. The UINT64_MAX + * sentinel preserves the existing unsigned generation-ring arithmetic. + * Otherwise, the checkpoint has incorporated every generation before + * first_generation. + */ + ctxt->generation_retired = + first_generation == 0 ? (uint64)-1 : first_generation - 1; ctxt->is_empty = TRUE; @@ -411,6 +430,18 @@ memtable_context_init(memtable_context *ctxt, return STATUS_OK; } +platform_status +memtable_context_init(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn process, + void *process_ctxt) +{ + return memtable_context_init_at_generation( + ctxt, hid, cc, cfg, process, process_ctxt, 0); +} + void memtable_context_deinit(memtable_context *ctxt) { diff --git a/src/memtable.h b/src/memtable.h index 7c82aadd..bfea7002 100644 --- a/src/memtable.h +++ b/src/memtable.h @@ -159,6 +159,17 @@ memtable_maybe_rotate_and_begin_insert(memtable_context *ctxt, void memtable_end_insert(memtable_context *ctxt); +/* + * Exclude all inserts, including an insert that has already acquired its + * shared insert lock. A checkpoint uses this around sealing the log prefix + * that describes its snapshot. Must be paired with memtable_unblock_inserts. + */ +void +memtable_block_inserts(memtable_context *ctxt); + +void +memtable_unblock_inserts(memtable_context *ctxt); + void memtable_begin_lookup(memtable_context *ctxt); @@ -211,6 +222,24 @@ memtable_context_init(memtable_context *ctxt, process_fn process, void *process_ctxt); +/* + * Initialize the reusable memtable ring at first_generation. Recovery passes + * the checkpoint's incorporated generation plus one; the trunk is understood + * to have incorporated all earlier generations. In particular, slot + * first_generation % max_memtables is the active memtable and the other slots + * represent the following logical generations. first_generation == 0 is the + * fresh-database case and has no incorporated generation. A caller must not + * wrap a checkpoint generation of UINT64_MAX into zero. + */ +platform_status +memtable_context_init_at_generation(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn process, + void *process_ctxt, + uint64 first_generation); + void memtable_context_deinit(memtable_context *ctxt); diff --git a/src/mini_allocator.c b/src/mini_allocator.c index 1099641f..6832604e 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -198,7 +198,6 @@ mini_full_lock_meta_tail(mini_allocator *mini) static void mini_full_unlock_meta_page(mini_allocator *mini, page_handle *meta_page) { - cache_mark_dirty(mini->cc, meta_page); cache_unlock(mini->cc, meta_page); cache_unclaim(mini->cc, meta_page); cache_unget(mini->cc, meta_page); @@ -761,35 +760,78 @@ mini_deinit(cache *cc, uint64 meta_head, page_type type) *----------------------------------------------------------------------------- * mini_for_each_meta_page -- * - * Calls func on each meta_page in the mini_allocator. + * Walks the meta_page chain of the mini_allocator, calling func on each + * meta_page. + * + * If before is non-NULL, it is called once per distinct meta extent -- + * before that extent's first page is cache_get()'d -- with just the + * extent's address. This lets a caller that does not yet have allocator + * ownership of these pages (e.g. crash recovery, rebuilding the refcount + * table from scratch) establish it first, before cache_get()'s debug + * allocation checks run. Ordinary callers, which already own everything + * they are walking, pass NULL. + * + * func and before may fail; the walk stops and returns the first failure. * * Results: - * None + * STATUS_OK, the first failure from before/func, or STATUS_IO_ERROR if a + * meta page could not be read. * * Side effects: - * func may store output in arg. + * func/before may store output in arg. *----------------------------------------------------------------------------- */ -typedef void (*mini_for_each_meta_page_fn)(cache *cc, - page_type type, - page_handle *meta_page, - void *arg); +typedef platform_status (*mini_meta_extent_fn)(cache *cc, + page_type type, + uint64 extent_addr, + void *arg); -static void +typedef platform_status (*mini_for_each_meta_page_fn)(cache *cc, + page_type type, + page_handle *meta_page, + void *arg); + +static platform_status mini_for_each_meta_page(cache *cc, uint64 meta_head, page_type type, + mini_meta_extent_fn before, mini_for_each_meta_page_fn func, void *arg) { - uint64 meta_addr = meta_head; + uint64 meta_addr = meta_head; + uint64 extent_size = cache_extent_size(cc); + uint64 prior_extent = (uint64)-1; + while (meta_addr != 0) { + if (before != NULL) { + uint64 extent_addr = meta_addr - meta_addr % extent_size; + if (extent_addr != prior_extent) { + platform_status rc = before(cc, type, extent_addr, arg); + if (!SUCCESS(rc)) { + return rc; + } + prior_extent = extent_addr; + } + } + page_handle *meta_page = cache_get(cc, meta_addr, TRUE, type); - func(cc, type, meta_page, arg); - meta_addr = mini_get_next_meta_addr(meta_page); + if (meta_page == NULL) { + return STATUS_IO_ERROR; + } + + platform_status rc = + func == NULL ? STATUS_OK : func(cc, type, meta_page, arg); + uint64 next_meta_addr = + SUCCESS(rc) ? mini_get_next_meta_addr(meta_page) : 0; cache_unget(cc, meta_page); + if (!SUCCESS(rc)) { + return rc; + } + meta_addr = next_meta_addr; } + return STATUS_OK; } /* mini_for_each(): call a function on each allocated extent in the @@ -805,7 +847,7 @@ typedef struct for_each_func { void *arg; } for_each_func; -static void +static platform_status mini_for_each_meta_page_func(cache *cc, page_type type, page_handle *meta_page, @@ -822,6 +864,7 @@ mini_for_each_meta_page_func(cache *cc, fef->arg); entry = next_entry(entry); } + return STATUS_OK; } static void @@ -833,7 +876,7 @@ mini_for_each(cache *cc, { for_each_func fef = {func, out}; mini_for_each_meta_page( - cc, meta_head, type, mini_for_each_meta_page_func, &fef); + cc, meta_head, type, NULL, mini_for_each_meta_page_func, &fef); } @@ -917,6 +960,270 @@ mini_prefetch(cache *cc, page_type type, uint64 meta_head) mini_for_each(cc, meta_head, type, mini_prefetch_extent, NULL); } +/* + * ----------------------------------------------------------------------------- + * mini_recover_allocations -- crash-recovery allocator-reference rebuild. + * ----------------------------------------------------------------------------- + */ + +static platform_status +mini_recovery_corruption(const char *reason, uint64 addr) +{ + platform_error_log( + "Malformed mini allocator metadata: %s (addr=%lu).\n", reason, addr); + return STATUS_INVALID_STATE; +} + +static bool32 +mini_recovery_valid_geometry(const allocator_config *cfg) +{ + if (cfg == NULL || cfg->io_cfg == NULL || cfg->capacity == 0 + || cfg->io_cfg->page_size == 0 || cfg->io_cfg->extent_size == 0 + || cfg->io_cfg->page_size < offsetof(mini_meta_hdr, entry_buffer) + || cfg->io_cfg->extent_size < cfg->io_cfg->page_size + || cfg->io_cfg->extent_size % cfg->io_cfg->page_size != 0 + || cfg->capacity % cfg->io_cfg->extent_size != 0) + { + return FALSE; + } + + return TRUE; +} + +static bool32 +mini_recovery_valid_page_addr(const allocator_config *cfg, uint64 addr) +{ + uint64 page_size = cfg->io_cfg->page_size; + + return addr != 0 && addr % page_size == 0 + && addr <= cfg->capacity - page_size; +} + +static bool32 +mini_recovery_valid_extent_addr(const allocator_config *cfg, uint64 addr) +{ + uint64 extent_size = cfg->io_cfg->extent_size; + + return addr != 0 && addr % extent_size == 0 + && addr <= cfg->capacity - extent_size; +} + +static platform_status +mini_recovery_validate_meta_header(const mini_meta_hdr *hdr, + uint64 page_size, + uint64 expected_prev, + uint64 meta_addr) +{ + uint64 first_entry_offset = offsetof(mini_meta_hdr, entry_buffer); + uint64 max_entries = (page_size - first_entry_offset) / sizeof(meta_entry); + if (hdr->prev_meta_addr != expected_prev) { + return mini_recovery_corruption("metadata prev link is not reciprocal", + meta_addr); + } + + if (hdr->num_entries > max_entries) { + return mini_recovery_corruption("metadata entry count exceeds page", + meta_addr); + } + + uint64 expected_pos = + first_entry_offset + (uint64)hdr->num_entries * sizeof(meta_entry); + if (hdr->pos != expected_pos) { + return mini_recovery_corruption("metadata entry position is invalid", + meta_addr); + } + + return STATUS_OK; +} + +static platform_status +mini_recovery_validate_next_meta_addr(const allocator_config *cfg, + uint64 meta_addr, + uint64 next_meta_addr) +{ + if (next_meta_addr == 0) { + return STATUS_OK; + } + + if (!mini_recovery_valid_page_addr(cfg, next_meta_addr)) { + return mini_recovery_corruption("metadata next link is not a page", + next_meta_addr); + } + + uint64 extent_size = cfg->io_cfg->extent_size; + uint64 page_size = cfg->io_cfg->page_size; + uint64 offset = meta_addr % extent_size; + if (offset == extent_size - page_size) { + if (next_meta_addr % extent_size != 0) { + return mini_recovery_corruption( + "metadata extent transition does not start at an extent base", + next_meta_addr); + } + } else if (next_meta_addr != meta_addr + page_size) { + return mini_recovery_corruption( + "metadata pages do not advance contiguously within an extent", + next_meta_addr); + } + + return STATUS_OK; +} + +typedef struct mini_recover_state { + allocator *al; + allocator_config *cfg; + uint64 expected_prev; + uint64 page_count; + uint64 max_meta_pages; +} mini_recover_state; + +/* + * The "before" hook mini_for_each_meta_page() calls once per distinct meta + * extent, before that extent's first page is cache_get()'d. Recording the + * reference here -- ahead of the read -- is what lets a rebuilding allocator + * (which does not yet consider this extent allocated) satisfy cache_get()'s + * debug allocation check. + */ +static platform_status +mini_recover_visit_meta_extent(cache *cc, + page_type type, + uint64 extent_addr, + void *arg) +{ + mini_recover_state *state = (mini_recover_state *)arg; + + if (!mini_recovery_valid_extent_addr(state->cfg, extent_addr)) { + return mini_recovery_corruption("metadata extent is out of range", + extent_addr); + } + return allocator_recovery_record_reference(state->al, extent_addr, type); +} + +/* + * The "func" hook mini_for_each_meta_page() calls once per meta page, with the + * page held. Validates the page and every data-extent entry on it before + * trusting any of it, and records a reference for each data extent found (its + * type is self-describing on disk, unlike a meta page's). + */ +static platform_status +mini_recover_visit_meta_page(cache *cc, + page_type type, + page_handle *meta_page, + void *arg) +{ + mini_recover_state *state = (mini_recover_state *)arg; + uint64 meta_addr = meta_page->disk_addr; + + if (state->page_count == state->max_meta_pages) { + return mini_recovery_corruption("metadata chain exceeds disk pages", + meta_addr); + } + state->page_count++; + + mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; + platform_status rc = mini_recovery_validate_meta_header( + hdr, state->cfg->io_cfg->page_size, state->expected_prev, meta_addr); + if (!SUCCESS(rc)) { + return rc; + } + + rc = mini_recovery_validate_next_meta_addr( + state->cfg, meta_addr, hdr->next_meta_addr); + if (!SUCCESS(rc)) { + return rc; + } + + meta_entry *entry = first_entry(meta_page); + for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { + uint64 batch = meta_entry_batch(entry); + page_type extent_type = meta_entry_type(entry); + uint64 extent_number = + entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); + + if (batch >= MINI_MAX_BATCHES || extent_type < PAGE_TYPE_FIRST + || extent_type >= NUM_PAGE_TYPES || extent_number == 0 + || extent_number > UINT64_MAX / state->cfg->io_cfg->extent_size) + { + return mini_recovery_corruption("metadata extent entry is invalid", + meta_addr); + } + + uint64 extent_addr = extent_number * state->cfg->io_cfg->extent_size; + if (!mini_recovery_valid_extent_addr(state->cfg, extent_addr)) { + return mini_recovery_corruption("metadata extent is out of range", + extent_addr); + } + + rc = allocator_recovery_record_reference( + state->al, extent_addr, extent_type); + if (!SUCCESS(rc)) { + return rc; + } + + entry = next_entry(entry); + } + + state->expected_prev = meta_addr; + return STATUS_OK; +} + +/* + * mini_recover_allocations -- + * + * Rebuild allocator references for a finalized mini allocator without + * trusting allocator refcounts -- the discovery primitive crash recovery + * uses to reconstruct them. Records one reference for every metadata + * extent and every data-extent entry found in the on-disk mini metadata + * stream (data entries are not deduplicated: repeated entries mean + * repeated references, and are recorded exactly as often as they occur). + * + * meta_type is the page type of meta_head's own chain; each data extent's + * type is read from its own metadata entry. + * + * Validates the metadata-page chain, page-header bounds, page types, + * batches, and extent addresses before using them. A malformed on-disk + * stream returns STATUS_INVALID_STATE. This function performs no writes + * and does not use allocator refcounts to decide what to traverse. + * + * This is a physical enumeration only. Recovering logical reference + * multiplicity, and deduplicating references shared by distinct mini + * allocator roots, remains the responsibility of the higher-level + * trunk/log recovery walker. + */ +platform_status +mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type) +{ + if (cc == NULL || meta_type < PAGE_TYPE_FIRST || meta_type >= NUM_PAGE_TYPES) + { + return STATUS_BAD_PARAM; + } + + allocator *al = cache_get_allocator(cc); + if (al == NULL) { + return STATUS_BAD_PARAM; + } + allocator_config *cfg = allocator_get_config(al); + if (!mini_recovery_valid_geometry(cfg)) { + return STATUS_BAD_PARAM; + } + if (!mini_recovery_valid_page_addr(cfg, meta_head)) { + return mini_recovery_corruption("metadata head is not a page", meta_head); + } + + mini_recover_state state = {.al = al, + .cfg = cfg, + .expected_prev = 0, + .page_count = 0, + .max_meta_pages = cfg->capacity + / cfg->io_cfg->page_size}; + + return mini_for_each_meta_page(cc, + meta_head, + meta_type, + mini_recover_visit_meta_extent, + mini_recover_visit_meta_page, + &state); +} + /* *----------------------------------------------------------------------------- * mini_meta_cursor -- cursor over a mini_allocator's extent entries. @@ -1080,7 +1387,7 @@ space_use_add_extent(cache *cc, page_type type, uint64 extent_addr, void *out) *sum += cache_extent_size(cc); } -static void +static platform_status space_use_add_meta_page(cache *cc, page_type type, page_handle *meta_page, @@ -1088,6 +1395,7 @@ space_use_add_meta_page(cache *cc, { uint64 *sum = (uint64 *)out; *sum += cache_page_size(cc); + return STATUS_OK; } uint64 @@ -1096,7 +1404,7 @@ mini_space_use_bytes(cache *cc, uint64 meta_head, page_type type) uint64 total = 0; mini_for_each(cc, meta_head, type, space_use_add_extent, &total); mini_for_each_meta_page( - cc, meta_head, type, space_use_add_meta_page, &total); + cc, meta_head, type, NULL, space_use_add_meta_page, &total); return total; } diff --git a/src/mini_allocator.h b/src/mini_allocator.h index 1a7995b7..5407530f 100644 --- a/src/mini_allocator.h +++ b/src/mini_allocator.h @@ -113,6 +113,33 @@ mini_unblock_dec_ref(cache *cc, uint64 meta_head); void mini_prefetch(cache *cc, page_type type, uint64 meta_head); +/* + * mini_recover_allocations -- + * + * Rebuild allocator references for a finalized mini allocator without + * trusting allocator refcounts. This is the discovery primitive crash + * recovery uses to reconstruct them: it records one allocator reference + * (via allocator_recovery_record_reference()) for every metadata extent + * and for every data-extent entry in the on-disk mini metadata stream. + * Data entries are deliberately not deduplicated: repeated entries + * represent repeated references in the mini allocator and are recorded + * exactly as often as they occur. meta_type is the page type of + * meta_head's own chain; each data extent's type is read from its own + * metadata entry, so it is not a parameter here. + * + * Validates the metadata-page chain, page-header bounds, page types, + * batches, and extent addresses before using them. A malformed on-disk + * stream returns STATUS_INVALID_STATE. This function performs no writes + * and does not use allocator refcounts to decide what to traverse. + * + * This is a physical enumeration only. Recovering logical reference + * multiplicity, and deduplicating references shared by distinct mini + * allocator roots, remains the responsibility of the higher-level + * trunk/log recovery walker. + */ +platform_status +mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type); + /* * mini_meta_cursor: a non-blocking cursor over the extent entries of a * finalized mini_allocator. Entries from all batches are interleaved in diff --git a/src/platform_linux/laio.c b/src/platform_linux/laio.c index 87e6c3f5..6988514b 100644 --- a/src/platform_linux/laio.c +++ b/src/platform_linux/laio.c @@ -213,6 +213,35 @@ laio_write(io_handle *ioh, void *buf, uint64 bytes, uint64 addr) return STATUS_IO_ERROR; } +/* + *-------------------------------------------------------------------------- + * laio_durable_barrier -- + * + * Make completed writes durable. Waiting for asynchronous I/O only tells us + * that the kernel accepted the writes; fdatasync() supplies the persistence + * ordering needed by checkpoint publication on buffered files. + *-------------------------------------------------------------------------- + */ +static platform_status +laio_durable_barrier(io_handle *ioh) +{ + laio_handle *io = (laio_handle *)ioh; + int ret; + + do { + ret = fdatasync(io->fd); + } while (ret != 0 && errno == EINTR); + + if (ret == 0) { + return STATUS_OK; + } + + int saved_errno = errno; + platform_error_log("laio_durable_barrier: fdatasync failed: %s\n", + strerror(saved_errno)); + return CONST_STATUS(saved_errno); +} + /* * Accessor method: Return opaque handle to IO-context setup by io_setup(). */ @@ -674,6 +703,7 @@ static io_ops laio_ops = { .async_state_init = laio_async_state_init, .cleanup = laio_cleanup, .wait_all = laio_wait_all, + .durable_barrier = laio_durable_barrier, .print_stats = laio_print_stats, .reset_stats = laio_reset_stats, }; diff --git a/src/platform_linux/platform_io.h b/src/platform_linux/platform_io.h index 1d96bf1a..cb60e861 100644 --- a/src/platform_linux/platform_io.h +++ b/src/platform_linux/platform_io.h @@ -78,6 +78,7 @@ typedef platform_status (*io_async_state_init_fn)(io_async_state *state, typedef void (*io_cleanup_fn)(io_handle *io, uint64 count); typedef void (*io_wait_all_fn)(io_handle *io); +typedef platform_status (*io_durable_barrier_fn)(io_handle *io); typedef void (*io_register_thread_fn)(io_handle *io); typedef void (*io_deregister_thread_fn)(io_handle *io); typedef bool32 (*io_max_latency_elapsed_fn)(io_handle *io, timestamp ts); @@ -95,6 +96,7 @@ typedef struct io_ops { io_async_state_init_fn async_state_init; io_cleanup_fn cleanup; io_wait_all_fn wait_all; + io_durable_barrier_fn durable_barrier; io_register_thread_fn register_thread; io_deregister_thread_fn deregister_thread; io_max_latency_elapsed_fn max_latency_elapsed; @@ -197,13 +199,34 @@ io_cleanup(io_handle *io, uint64 count) return io->ops->cleanup(io, count); } -// Guarantees all in-flight IOs are complete before return +// Wait until we have seen each process's I/O be quiescent. +// +// This means that all I/Os that were in-flight at the start of this call are +// guaranteed to have finished before this call returns. +// +// But I/Os that are issued after this call starts may not be complete when +// this call returns. static inline void io_wait_all(io_handle *io) { return io->ops->wait_all(io); } +/* + *-------------------------------------------------------------------------- + * io_durable_barrier + * + * Ensure that writes which completed before this call survive a power loss. + * This is deliberately distinct from io_wait_all(), which only waits for + * asynchronous I/O completion. + *-------------------------------------------------------------------------- + */ +static inline platform_status +io_durable_barrier(io_handle *io) +{ + return io->ops->durable_barrier(io); +} + static inline void io_register_thread(io_handle *io) { diff --git a/src/rc_allocator.c b/src/rc_allocator.c index 70257dc5..e86d8519 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -17,7 +17,15 @@ #include "platform_typed_alloc.h" #include "poison.h" -#define RC_ALLOCATOR_META_PAGE_CSUM_SEED (2718281828) +#define RC_ALLOCATOR_META_PAGE_CSUM_SEED (2718281828) +#define RC_ALLOCATOR_CLEAN_STATE_CSUM_SEED (2718281829) + +#define RC_ALLOCATOR_FORMAT_MAGIC (0x534442414C4C4F43ULL) // SDBALLOC +#define RC_ALLOCATOR_FORMAT_VERSION (1) + +#define RC_ALLOCATOR_CLEAN_STATE_MAGIC (0x534442434C45414EULL) // SDBCLEAN +#define RC_ALLOCATOR_CLEAN_STATE_VERSION (1) +#define RC_ALLOCATOR_CLEAN_STATE_SLOTS (2) /* * Base offset from where the allocator starts. Currently hard coded to 0. @@ -35,6 +43,28 @@ */ #define SHOULD_TRACE(addr) (0) // Do not trace anything +/* + * A/B clean-state records live in two fixed extents after the refcount map. + * They are deliberately separate from the sole allocator bootstrap page: a + * torn state update must leave an older valid state record available for the + * next mount or rebuild. + */ +typedef struct ONDISK rc_allocator_clean_state { + uint64 magic; + uint64 format_version; + uint64 sequence; + bool32 clean_shutdown; + checksum128 checksum; +} rc_allocator_clean_state; + +typedef struct rc_allocator_clean_states { + rc_allocator_clean_state state[RC_ALLOCATOR_CLEAN_STATE_SLOTS]; + bool32 valid[RC_ALLOCATOR_CLEAN_STATE_SLOTS]; + bool32 have_newest; + uint64 newest_slot; + bool32 duplicate_sequence; +} rc_allocator_clean_states; + /* *------------------------------------------------------------------------------ * Function declarations and virtual trampolines @@ -93,6 +123,20 @@ rc_allocator_get_ref_virtual(allocator *a, uint64 addr) return rc_allocator_get_ref(al, addr); } +platform_status +rc_allocator_rebuild_acquire_extent(rc_allocator *al, + uint64 extent_addr, + page_type type); + +platform_status +rc_allocator_recovery_record_reference_virtual(allocator *a, + uint64 addr, + page_type type) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_rebuild_acquire_extent(al, addr, type); +} + platform_status rc_allocator_get_super_addr(rc_allocator *al, allocator_root_id spl_id, @@ -187,6 +231,7 @@ const static allocator_ops rc_allocator_ops = { .inc_ref = rc_allocator_inc_ref_virtual, .dec_ref = rc_allocator_dec_ref_virtual, .get_ref = rc_allocator_get_ref_virtual, + .recovery_record_reference = rc_allocator_recovery_record_reference_virtual, .get_super_addr = rc_allocator_get_super_addr_virtual, .alloc_super_addr = rc_allocator_alloc_super_addr_virtual, .remove_super_addr = rc_allocator_remove_super_addr_virtual, @@ -231,6 +276,16 @@ rc_allocator_meta_page_checksum(const rc_allocator_meta_page *meta_page) RC_ALLOCATOR_META_PAGE_CSUM_SEED); } +static platform_status +rc_allocator_write_meta_page(rc_allocator *al) +{ + al->meta_page->checksum = rc_allocator_meta_page_checksum(al->meta_page); + return io_write(al->io, + al->meta_page, + al->cfg->io_cfg->page_size, + RC_ALLOCATOR_BASE_OFFSET); +} + static disk_geometry rc_allocator_config_get_disk_geometry(allocator_config *cfg) { @@ -241,6 +296,256 @@ rc_allocator_config_get_disk_geometry(allocator_config *cfg) }; } +static uint64 +rc_allocator_refcount_buffer_size(const allocator_config *cfg) +{ + uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); + return ROUNDUP(buffer_size, cfg->io_cfg->page_size); +} + +static uint64 +rc_allocator_refcount_extent_count(const allocator_config *cfg) +{ + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); + return (buffer_size + cfg->io_cfg->extent_size - 1) + / cfg->io_cfg->extent_size; +} + +static uint64 +rc_allocator_clean_state_extent_no(const allocator_config *cfg, uint64 slot) +{ + platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); + return 1 + rc_allocator_refcount_extent_count(cfg) + slot; +} + +static uint64 +rc_allocator_clean_state_addr(const allocator_config *cfg, uint64 slot) +{ + return rc_allocator_clean_state_extent_no(cfg, slot) + * cfg->io_cfg->extent_size; +} + +static uint64 +rc_allocator_reserved_extent_count(const allocator_config *cfg) +{ + return 1 + rc_allocator_refcount_extent_count(cfg) + + RC_ALLOCATOR_CLEAN_STATE_SLOTS; +} + +static checksum128 +rc_allocator_clean_state_checksum(const rc_allocator_clean_state *state) +{ + return platform_checksum128(state, + offsetof(rc_allocator_clean_state, checksum), + RC_ALLOCATOR_CLEAN_STATE_CSUM_SEED); +} + +static bool32 +rc_allocator_clean_state_is_valid(const rc_allocator_clean_state *state) +{ + return state->magic == RC_ALLOCATOR_CLEAN_STATE_MAGIC + && state->format_version == RC_ALLOCATOR_CLEAN_STATE_VERSION + && state->sequence != 0 + && (state->clean_shutdown == FALSE || state->clean_shutdown == TRUE) + && platform_checksum_is_equal( + state->checksum, rc_allocator_clean_state_checksum(state)); +} + +static platform_status +rc_allocator_read_clean_state(rc_allocator *al, + uint64 slot, + rc_allocator_clean_state *state, + bool32 *valid) +{ + platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); + platform_assert(sizeof(*state) <= al->cfg->io_cfg->page_size); + + buffer_handle buffer; + platform_status rc = + platform_buffer_init(&buffer, al->cfg->io_cfg->page_size); + if (!SUCCESS(rc)) { + return rc; + } + + void *page = platform_buffer_getaddr(&buffer); + rc = io_read(al->io, + page, + al->cfg->io_cfg->page_size, + rc_allocator_clean_state_addr(al->cfg, slot)); + if (SUCCESS(rc)) { + memcpy(state, page, sizeof(*state)); + *valid = rc_allocator_clean_state_is_valid(state); + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +static platform_status +rc_allocator_write_clean_state(rc_allocator *al, + uint64 slot, + const rc_allocator_clean_state *state, + bool32 durable) +{ + platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); + platform_assert(sizeof(*state) <= al->cfg->io_cfg->page_size); + + buffer_handle buffer; + platform_status rc = + platform_buffer_init(&buffer, al->cfg->io_cfg->page_size); + if (!SUCCESS(rc)) { + return rc; + } + + void *page = platform_buffer_getaddr(&buffer); + memset(page, 0, al->cfg->io_cfg->page_size); + memcpy(page, state, sizeof(*state)); + rc = io_write(al->io, + page, + al->cfg->io_cfg->page_size, + rc_allocator_clean_state_addr(al->cfg, slot)); + if (SUCCESS(rc) && durable) { + rc = io_durable_barrier(al->io); + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +static platform_status +rc_allocator_load_clean_states(rc_allocator *al, + rc_allocator_clean_states *states) +{ + ZERO_CONTENTS(states); + for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { + platform_status rc = rc_allocator_read_clean_state( + al, slot, &states->state[slot], &states->valid[slot]); + if (!SUCCESS(rc)) { + return rc; + } + + if (!states->valid[slot]) { + continue; + } + if (!states->have_newest + || states->state[states->newest_slot].sequence + < states->state[slot].sequence) + { + states->have_newest = TRUE; + states->newest_slot = slot; + } else if (states->state[states->newest_slot].sequence + == states->state[slot].sequence) + { + states->duplicate_sequence = TRUE; + } + } + return STATUS_OK; +} + +static platform_status +rc_allocator_publish_clean_state(rc_allocator *al, bool32 clean_shutdown) +{ + rc_allocator_clean_states states; + platform_status rc = rc_allocator_load_clean_states(al, &states); + if (!SUCCESS(rc)) { + return rc; + } + + if (states.have_newest + && states.state[states.newest_slot].sequence == UINT64_MAX) + { + return STATUS_LIMIT_EXCEEDED; + } + + uint64 target_slot = states.have_newest ? states.newest_slot ^ 1 : 0; + rc_allocator_clean_state state; + ZERO_CONTENTS(&state); + state.magic = RC_ALLOCATOR_CLEAN_STATE_MAGIC; + state.format_version = RC_ALLOCATOR_CLEAN_STATE_VERSION; + state.sequence = + states.have_newest ? states.state[states.newest_slot].sequence + 1 : 1; + state.clean_shutdown = clean_shutdown; + state.checksum = rc_allocator_clean_state_checksum(&state); + return rc_allocator_write_clean_state(al, target_slot, &state, TRUE); +} + +static platform_status +rc_allocator_initialize_clean_states(rc_allocator *al) +{ + for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { + rc_allocator_clean_state state; + ZERO_CONTENTS(&state); + state.magic = RC_ALLOCATOR_CLEAN_STATE_MAGIC; + state.format_version = RC_ALLOCATOR_CLEAN_STATE_VERSION; + state.sequence = slot + 1; + state.clean_shutdown = FALSE; + state.checksum = rc_allocator_clean_state_checksum(&state); + + platform_status rc = + rc_allocator_write_clean_state(al, slot, &state, FALSE); + if (!SUCCESS(rc)) { + return rc; + } + } + return io_durable_barrier(al->io); +} + +static void +rc_allocator_record_allocated_extent(rc_allocator *al) +{ + int64 curr_allocated = __sync_add_and_fetch(&al->stats.curr_allocated, 1); + int64 max_allocated = al->stats.max_allocated; + while (curr_allocated > max_allocated) { + __sync_bool_compare_and_swap( + &al->stats.max_allocated, max_allocated, curr_allocated); + max_allocated = al->stats.max_allocated; + } +} + +static platform_status +rc_allocator_recovery_initialize_refcounts(rc_allocator *al) +{ + uint64 reserved_extent_count = rc_allocator_reserved_extent_count(al->cfg); + + if (reserved_extent_count > al->cfg->extent_capacity) { + platform_error_log("Allocator needs %lu reserved extents, but its " + "configured capacity is only %lu extents.\n", + reserved_extent_count, + al->cfg->extent_capacity); + return STATUS_BAD_PARAM; + } + + memset(al->ref_count, 0, rc_allocator_refcount_buffer_size(al->cfg)); + + /* + * Extent 0 contains both the allocator meta page and every fixed table + * superblock. The refcount table begins at extent 1; the two extents + * after it hold alternating clean-state records. + */ + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) { + platform_assert(al->ref_count[extent_no] == AL_FREE); + al->ref_count[extent_no] = AL_ONE_REF; + rc_allocator_record_allocated_extent(al); + } + + /* Match a regular mount: the first post-recovery allocation scans at 0. */ + al->hand = 0; + return STATUS_OK; +} + +static bool32 +rc_allocator_recovery_extent_is_reserved(const rc_allocator *al, + uint64 extent_no) +{ + return extent_no < rc_allocator_reserved_extent_count(al->cfg); +} + static platform_status rc_allocator_validate_disk_geometry(rc_allocator *al) { @@ -309,7 +614,9 @@ rc_allocator_init_meta_page(rc_allocator *al) memset(al->meta_page->splinters, INVALID_ALLOCATOR_ROOT_ID, sizeof(al->meta_page->splinters)); - al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); + al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); + al->meta_page->format_magic = RC_ALLOCATOR_FORMAT_MAGIC; + al->meta_page->format_version = RC_ALLOCATOR_FORMAT_VERSION; return STATUS_OK; } @@ -344,6 +651,15 @@ rc_allocator_valid_config(allocator_config *cfg) return STATUS_BAD_PARAM; } + if (rc_allocator_reserved_extent_count(cfg) > cfg->extent_capacity) { + platform_error_log("Configured allocator has %lu extents, but needs %lu " + "reserved extents for metadata, refcounts, and clean " + "state.\n", + cfg->extent_capacity, + rc_allocator_reserved_extent_count(cfg)); + return STATUS_BAD_PARAM; + } + // Assert: Disk size == (page-size * #-of-pages) if (cfg->capacity != (cfg->io_cfg->page_size * cfg->page_capacity)) { platform_error_log("Configured disk size, %lu bytes, is not an integral" @@ -410,8 +726,7 @@ rc_allocator_init(rc_allocator *al, return rc; } // To ensure alignment always allocate in multiples of page size. - uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); - buffer_size = ROUNDUP(buffer_size, cfg->io_cfg->page_size); + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); rc = platform_buffer_init(&al->bh, buffer_size); if (!SUCCESS(rc)) { platform_mutex_destroy(&al->lock); @@ -431,14 +746,42 @@ rc_allocator_init(rc_allocator *al, * Allocate room for the ref counts, use same rounded up size used in buffer * creation. */ - rc_extent_count = (buffer_size + al->cfg->io_cfg->extent_size - 1) - / al->cfg->io_cfg->extent_size; + rc_extent_count = rc_allocator_refcount_extent_count(cfg); for (uint64 i = 0; i < rc_extent_count; i++) { allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); platform_assert(addr == cfg->io_cfg->extent_size * (i + 1)); } + for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { + allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); + platform_assert(addr == rc_allocator_clean_state_addr(cfg, slot)); + } + + /* + * Persist the immutable bootstrap layout and both initial false state + * records before returning a newly created allocator. A crash before a + * later clean close therefore enters rebuild recovery rather than trusting + * this freshly initialized refcount map. + */ + rc = rc_allocator_initialize_clean_states(al); + if (!SUCCESS(rc)) { + goto deinit_allocator; + } + rc = rc_allocator_write_meta_page(al); + if (!SUCCESS(rc)) { + goto deinit_allocator; + } + rc = io_durable_barrier(al->io); + if (!SUCCESS(rc)) { + goto deinit_allocator; + } + return STATUS_OK; + +deinit_allocator: + rc_allocator_deinit(al); + ZERO_CONTENTS(al); + return rc; } void @@ -458,12 +801,13 @@ rc_allocator_deinit(rc_allocator *al) * Write the file system to disk *---------------------------------------------------------------------- */ -platform_status -rc_allocator_mount(rc_allocator *al, - allocator_config *cfg, - io_handle *io, - platform_heap_id hid, - platform_module_id mid) +static platform_status +rc_allocator_mount_internal(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid, + bool32 rebuild_refcounts) { platform_status status; @@ -493,8 +837,7 @@ rc_allocator_mount(rc_allocator *al, platform_assert(cfg->capacity == cfg->io_cfg->page_size * cfg->page_capacity); - uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); - buffer_size = ROUNDUP(buffer_size, cfg->io_cfg->page_size); + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); status = platform_buffer_init(&al->bh, buffer_size); if (!SUCCESS(status)) { platform_free(al->heap_id, al->meta_page); @@ -508,45 +851,183 @@ rc_allocator_mount(rc_allocator *al, status = io_read( io, al->meta_page, al->cfg->io_cfg->page_size, RC_ALLOCATOR_BASE_OFFSET); if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return status; + goto deinit_buffer; } status = rc_allocator_validate_disk_geometry(al); if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return status; + goto deinit_buffer; } // validate the checksum of the meta page. checksum128 currChecksum = rc_allocator_meta_page_checksum(al->meta_page); if (!platform_checksum_is_equal(al->meta_page->checksum, currChecksum)) { platform_error_log("Corrupt SplinterDB allocator meta page on mount\n"); - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return STATUS_BAD_PARAM; + status = STATUS_BAD_PARAM; + goto deinit_buffer; } - // load the ref counts from disk. - status = io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); - if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return status; + if (al->meta_page->format_magic != RC_ALLOCATOR_FORMAT_MAGIC + || al->meta_page->format_version != RC_ALLOCATOR_FORMAT_VERSION) + { + platform_error_log("Unsupported SplinterDB allocator bootstrap format " + "on mount.\n"); + status = STATUS_BAD_PARAM; + goto deinit_buffer; } - for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { - if (al->ref_count[i] != 0) { - al->stats.curr_allocated++; + if (!rebuild_refcounts) { + rc_allocator_clean_states states; + status = rc_allocator_load_clean_states(al, &states); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + if (!states.have_newest || states.duplicate_sequence + || !states.state[states.newest_slot].clean_shutdown) + { + platform_error_log("Allocator was not cleanly shut down; recovery " + "rebuild is required.\n"); + status = STATUS_INVALID_STATE; + goto deinit_buffer; + } + } + + if (rebuild_refcounts) { + /* Do not leave a stale clean state if recovery itself is interrupted. */ + status = rc_allocator_publish_clean_state(al, FALSE); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + status = rc_allocator_recovery_initialize_refcounts(al); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + al->recovery_in_progress = TRUE; + } else { + // Load the ref counts from disk during a normal, clean mount. + status = + io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + + for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { + if (al->ref_count[i] != 0) { + al->stats.curr_allocated++; + } + } + + /* Mark dirty before handing the trusted map to any mutating caller. */ + status = rc_allocator_publish_clean_state(al, FALSE); + if (!SUCCESS(status)) { + goto deinit_buffer; } } return STATUS_OK; + +deinit_buffer: + platform_buffer_deinit(&al->bh); + al->ref_count = NULL; + platform_free(al->heap_id, al->meta_page); + al->meta_page = NULL; + platform_mutex_destroy(&al->lock); + return status; +} + +platform_status +rc_allocator_mount(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid) +{ + return rc_allocator_mount_internal(al, cfg, io, hid, mid, FALSE); +} + +platform_status +rc_allocator_mount_recovery(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid) +{ + return rc_allocator_mount_internal(al, cfg, io, hid, mid, TRUE); +} + +platform_status +rc_allocator_rebuild_acquire_extent(rc_allocator *al, + uint64 extent_addr, + page_type type) +{ + if (!al->recovery_in_progress) { + platform_error_log("Cannot acquire allocator extent while recovery is " + "not in progress.\n"); + return STATUS_INVALID_STATE; + } + + uint64 extent_size = al->cfg->io_cfg->extent_size; + if (extent_addr >= al->cfg->capacity || extent_addr % extent_size != 0) { + platform_error_log("Invalid allocator recovery extent address %lu.\n", + extent_addr); + return STATUS_BAD_PARAM; + } + + uint64 extent_no = rc_allocator_extent_number(al, extent_addr); + platform_assert(extent_no < al->cfg->extent_capacity); + if (rc_allocator_recovery_extent_is_reserved(al, extent_no)) { + platform_error_log("Cannot rebuild ownership of reserved allocator " + "extent %lu.\n", + extent_no); + return STATUS_BAD_PARAM; + } + + while (TRUE) { + refcount old_ref = + __atomic_load_n(&al->ref_count[extent_no], __ATOMIC_RELAXED); + if (old_ref == (refcount)-1) { + platform_error_log("Allocator recovery refcount overflow for extent " + "%lu.\n", + extent_no); + return STATUS_LIMIT_EXCEEDED; + } + + refcount new_ref = old_ref == AL_FREE ? AL_ONE_REF : old_ref + 1; + if (!__sync_bool_compare_and_swap( + &al->ref_count[extent_no], old_ref, new_ref)) + { + continue; + } + + if (old_ref == AL_FREE) { + platform_assert(type != PAGE_TYPE_INVALID); + rc_allocator_record_allocated_extent(al); + __sync_add_and_fetch(&al->stats.extent_allocs[type], 1); + } + return STATUS_OK; + } +} + +void +rc_allocator_rebuild_finish(rc_allocator *al) +{ + platform_assert(al != NULL); + platform_assert(al->recovery_in_progress); + + /* + * Intentionally no I/O here. A rebuilt map is durable only after a later + * clean unmount; another crash before then simply rebuilds it again. + */ + al->recovery_in_progress = FALSE; +} + +void +rc_allocator_abort_recovery(rc_allocator *al) +{ + platform_assert(al != NULL); + platform_assert(al->recovery_in_progress); + + rc_allocator_deinit(al); + ZERO_CONTENTS(al); } @@ -555,12 +1036,32 @@ rc_allocator_unmount(rc_allocator *al) { platform_status status; + if (al->recovery_in_progress) { + platform_error_log("Discarding incomplete allocator recovery instead of " + "persisting its partial refcount map.\n"); + rc_allocator_abort_recovery(al); + return; + } + // persist the ref counts upon unmount. - uint64 buffer_size = al->cfg->extent_capacity * sizeof(refcount); + uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); uint32 io_size = ROUNDUP(buffer_size, al->cfg->io_cfg->page_size); status = io_write(al->io, al->ref_count, io_size, al->cfg->io_cfg->extent_size); platform_assert_status_ok(status); + + /* + * This is the sole normal persistence point for the allocator map. The + * checkpoint record was already made durable by core_unmount(); do not + * advertise a clean allocator snapshot until this write has crossed the + * device durability boundary as well. + */ + status = io_durable_barrier(al->io); + platform_assert_status_ok(status); + + /* Publish clean permission in the alternate durable state record. */ + status = rc_allocator_publish_clean_state(al, TRUE); + platform_assert_status_ok(status); rc_allocator_deinit(al); } @@ -686,12 +1187,7 @@ rc_allocator_alloc_super_addr(rc_allocator *al, // assign the first available slot and update the on disk metadata. al->meta_page->splinters[idx] = allocator_root_id; *addr = (1 + idx) * al->cfg->io_cfg->page_size; - al->meta_page->checksum = - rc_allocator_meta_page_checksum(al->meta_page); - platform_status io_status = io_write(al->io, - al->meta_page, - al->cfg->io_cfg->page_size, - RC_ALLOCATOR_BASE_OFFSET); + platform_status io_status = rc_allocator_write_meta_page(al); platform_assert_status_ok(io_status); status = STATUS_OK; break; @@ -715,12 +1211,7 @@ rc_allocator_remove_super_addr(rc_allocator *al, */ if (al->meta_page->splinters[idx] == allocator_root_id) { al->meta_page->splinters[idx] = INVALID_ALLOCATOR_ROOT_ID; - al->meta_page->checksum = - rc_allocator_meta_page_checksum(al->meta_page); - platform_status status = io_write(al->io, - al->meta_page, - al->cfg->io_cfg->page_size, - RC_ALLOCATOR_BASE_OFFSET); + platform_status status = rc_allocator_write_meta_page(al); platform_assert_status_ok(status); platform_mutex_unlock(&al->lock); return; diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 06ed7256..b7ada5cc 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -33,7 +33,14 @@ *---------------------------------------------------------------------- */ typedef struct ONDISK rc_allocator_meta_page { - disk_geometry geometry; + disk_geometry geometry; + /* + * Identifies the immutable bootstrap layout. In particular, it proves + * that the two fixed clean-state extents after the refcount map are owned + * by this allocator rather than by an older on-disk format. + */ + uint64 format_magic; + uint64 format_version; allocator_root_id splinters[RC_ALLOCATOR_MAX_ROOT_IDS]; checksum128 checksum; } rc_allocator_meta_page; @@ -76,6 +83,13 @@ typedef struct rc_allocator { platform_mutex lock; platform_heap_id heap_id; + /* + * True between rc_allocator_mount_recovery() and either + * rc_allocator_rebuild_finish() or rc_allocator_abort_recovery(). An + * incomplete rebuilt map must never be written back to disk. + */ + bool32 recovery_in_progress; + // Stats -- not distributed for now rc_allocator_stats stats; } rc_allocator; @@ -90,6 +104,14 @@ rc_allocator_init(rc_allocator *al, void rc_allocator_deinit(rc_allocator *al); +/* + * Normal mount accepts only a durable clean-state record and publishes an + * unclean state record before returning. Those records occupy two fixed, + * alternating extents after the persisted refcount map, so normal lifecycle + * operations never rewrite the sole allocator bootstrap page. + * STATUS_INVALID_STATE means callers must use the recovery-rebuild path + * instead of trusting the persisted refcount map. + */ platform_status rc_allocator_mount(rc_allocator *al, allocator_config *cfg, @@ -97,6 +119,46 @@ rc_allocator_mount(rc_allocator *al, platform_heap_id hid, platform_module_id mid); +/* + * Mount the allocator for crash recovery. This validates and retains the + * allocator metadata page, but deliberately ignores the persisted refcount + * table. The caller must rebuild the in-memory table from durable objects, + * then call rc_allocator_rebuild_finish() before using normal allocator + * lifecycle operations. + */ +platform_status +rc_allocator_mount_recovery(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid); + +/* + * Add one logical ownership reference for an extent while rebuilding a + * recovery map. The first reference establishes the allocator's nonzero + * allocation floor (AL_ONE_REF) and records it in stats.extent_allocs[type]; + * later references increment the refcount normally. extent_addr must be the + * base address of a non-reserved allocator extent. + */ +platform_status +rc_allocator_rebuild_acquire_extent(rc_allocator *al, + uint64 extent_addr, + page_type type); + +/* + * Complete a successful rebuild without performing I/O. A later normal + * rc_allocator_unmount() may persist the rebuilt table on clean shutdown. + */ +void +rc_allocator_rebuild_finish(rc_allocator *al); + +/* + * Discard a partially rebuilt recovery map. Unlike rc_allocator_unmount(), + * this never writes allocator state to disk. + */ +void +rc_allocator_abort_recovery(rc_allocator *al); + platform_status rc_allocator_read_disk_geometry(const char *filename, disk_geometry *geometry); diff --git a/src/shard_log.c b/src/shard_log.c index c0eee131..1db98edd 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -26,7 +26,19 @@ static uint64 shard_log_magic_idx = 0; int -shard_log_write(log_handle *log, key tuple_key, message msg, uint64 generation); +shard_log_write(log_handle *log, + key tuple_key, + message msg, + uint64 memtable_generation, + uint64 leaf_generation); +platform_status +shard_log_seal(log_handle *log); +platform_status +shard_log_rotate(log_handle *log, + log_segment_info *sealed, + log_segment_info *fresh); +void +shard_log_release(log_handle *log); uint64 shard_log_addr(log_handle *log); uint64 @@ -36,6 +48,9 @@ shard_log_magic(log_handle *log); static log_ops shard_log_ops = { .write = shard_log_write, + .seal = shard_log_seal, + .rotate = shard_log_rotate, + .release = shard_log_release, .addr = shard_log_addr, .meta_addr = shard_log_meta_addr, .magic = shard_log_magic, @@ -93,6 +108,9 @@ page_handle * shard_log_alloc(shard_log *log, uint64 *next_extent) { uint64 addr = mini_alloc_page(&log->mini, 0, next_extent); + if (addr == 0) { + return NULL; + } return cache_alloc(log->cc, addr, PAGE_TYPE_LOG); } @@ -142,7 +160,15 @@ shard_log_zap(shard_log *log) thread_data->offset = 0; } - mini_dec_ref(cc, log->meta_head, PAGE_TYPE_LOG); + if (log->meta_head != 0) { + /* Drop unused per-batch reserves before releasing the mini root. */ + mini_release(&log->mini); + refcount ref = mini_dec_ref(cc, log->meta_head, PAGE_TYPE_LOG); + platform_assert(ref == 0); + log->meta_head = 0; + log->addr = 0; + log->has_pages = FALSE; + } } /* @@ -152,11 +178,12 @@ shard_log_zap(shard_log *log) * ------------------------------------------------------------------------- */ struct ONDISK log_entry { - uint64 generation; + uint64 memtable_generation; + uint64 leaf_generation; ondisk_tuple tuple; }; -#define INVALID_GENERATION ((uint64) - 1) +#define INVALID_LOG_GENERATION ((uint64) - 1) static key log_entry_key(log_entry *le) @@ -200,7 +227,19 @@ static bool32 terminal_log_entry(shard_log_config *cfg, char *page, log_entry *le) { return page + shard_log_page_size(cfg) - (char *)le < sizeof(log_entry) - || le->generation == INVALID_GENERATION; + || le->memtable_generation == INVALID_LOG_GENERATION; +} + +static inline void +log_entry_set_terminal(log_entry *le) +{ + /* + * terminal_log_entry() tests memtable_generation. Mark both generation + * fields invalid so the terminator cannot be confused with a record by + * diagnostics or a future format validator. + */ + le->leaf_generation = INVALID_LOG_GENERATION; + le->memtable_generation = INVALID_LOG_GENERATION; } static log_entry * @@ -216,20 +255,30 @@ get_new_page_for_thread(shard_log *log, { uint64 next_extent; - *page = shard_log_alloc(log, &next_extent); + *page = shard_log_alloc(log, &next_extent); + if (*page == NULL) { + return -1; + } thread_data->addr = (*page)->disk_addr; shard_log_hdr *hdr = (shard_log_hdr *)(*page)->data; hdr->magic = log->magic; hdr->next_extent_addr = next_extent; hdr->num_entries = 0; thread_data->offset = sizeof(shard_log_hdr); + log->has_pages = TRUE; return 0; } int -shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) +shard_log_write(log_handle *logh, + key tuple_key, + message msg, + uint64 memtable_generation, + uint64 leaf_generation) { debug_assert(key_is_user_key(tuple_key)); + debug_assert(memtable_generation != INVALID_LOG_GENERATION); + debug_assert(leaf_generation != INVALID_LOG_GENERATION); shard_log *log = (shard_log *)logh; cache *cc = log->cc; @@ -290,13 +339,13 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) if (free_space < new_entry_size) { if (sizeof(log_entry) <= free_space) { - cursor->generation = INVALID_GENERATION; + log_entry_set_terminal(cursor); } hdr->checksum = shard_log_checksum(log->cfg, page); cache_unlock(cc, page); cache_unclaim(cc, page); - cache_page_sync(cc, page, FALSE, PAGE_TYPE_LOG); + cache_page_writeback(cc, page, FALSE, PAGE_TYPE_LOG); cache_unget(cc, page); if (get_new_page_for_thread(log, thread_data, &page)) { @@ -309,7 +358,8 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) hdr = (shard_log_hdr *)page->data; } - cursor->generation = generation; + cursor->memtable_generation = memtable_generation; + cursor->leaf_generation = leaf_generation; copy_tuple_to_ondisk_tuple(&cursor->tuple, tuple_key, msg); hdr->num_entries++; @@ -332,6 +382,149 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) return 0; } +/* + * shard_log_seal -- + * + * Finalize every currently active per-thread append page. This is + * deliberately bounded by MAX_THREADS: it does not walk the historical + * log. A final terminal record (where there is room) and checksum make + * each page readable by shard_log_iterator_init(), then clearing the + * append cursor ensures a subsequent writer allocates a new page instead + * of changing the sealed one. + * + * The caller must prevent concurrent shard_log_write() and seal calls. + * In particular, thread_data is otherwise only protected by the + * per-thread writer convention, not by a log-wide lock. This function + * intentionally does not issue writeback: after establishing that + * exclusion, the caller takes cache_writeback_dirty(), followed by a + * durable barrier. + */ +platform_status +shard_log_seal(log_handle *logh) +{ + shard_log *log = (shard_log *)logh; + cache *cc = log->cc; + + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(log, thr_i); + uint64 addr = thread_data->addr; + if (addr == SHARD_UNMAPPED) { + continue; + } + + page_handle *page = cache_get(cc, addr, TRUE, PAGE_TYPE_LOG); + uint64 wait = 1; + while (!cache_try_claim(cc, page)) { + cache_unget(cc, page); + platform_sleep_ns(wait); + wait = wait > 1024 ? wait : 2 * wait; + page = cache_get(cc, addr, TRUE, PAGE_TYPE_LOG); + } + cache_lock(cc, page); + + debug_assert(thread_data->addr == addr); + debug_assert(thread_data->offset >= sizeof(shard_log_hdr)); + debug_assert(thread_data->offset <= shard_log_page_size(log->cfg)); + + shard_log_hdr *hdr = (shard_log_hdr *)page->data; + log_entry *cursor = (log_entry *)(page->data + thread_data->offset); + uint64 free_space = shard_log_page_size(log->cfg) - thread_data->offset; + if (sizeof(log_entry) <= free_space) { + log_entry_set_terminal(cursor); + } + hdr->checksum = shard_log_checksum(log->cfg, page); + + cache_unlock(cc, page); + cache_unclaim(cc, page); + cache_unget(cc, page); + + /* Subsequent writes must allocate a new append page. */ + thread_data->addr = SHARD_UNMAPPED; + thread_data->offset = 0; + } + + return STATUS_OK; +} + +/* + * Detach a sealed stream from the live log. The original allocation reference + * on its metadata extent is transferred to sealed; an eventual durable + * descriptor must own that reference. A fresh mini allocator is prepared + * before the old stream is sealed, so core can publish the fresh identity + * while inserts remain excluded. This only prepares a physical boundary; it + * does not publish or advance the logical durable-log tail. + */ +platform_status +shard_log_rotate(log_handle *logh, + log_segment_info *sealed, + log_segment_info *fresh_info) +{ + shard_log *log = (shard_log *)logh; + shard_log fresh; + + platform_assert(sealed != NULL); + platform_assert(fresh_info != NULL); + ZERO_CONTENTS(sealed); + ZERO_CONTENTS(fresh_info); + + platform_status rc = shard_log_init(&fresh, log->cc, log->cfg); + if (!SUCCESS(rc)) { + return rc; + } + + rc = shard_log_seal(logh); + if (!SUCCESS(rc)) { + shard_log_zap(&fresh); + return rc; + } + + /* No future allocation may use the old mini allocator. */ + mini_release(&log->mini); + + if (log->has_pages) { + *sealed = (log_segment_info){ + .addr = log->addr, + .meta_addr = log->meta_head, + .magic = log->magic, + }; + } else { + /* An empty segment has no descriptor and no retained ownership. */ + refcount ref = mini_dec_ref(log->cc, log->meta_head, PAGE_TYPE_LOG); + platform_assert(ref == 0); + } + + log->mini = fresh.mini; + log->addr = fresh.addr; + log->meta_head = fresh.meta_head; + log->magic = fresh.magic; + log->has_pages = FALSE; + memcpy(log->thread_data, fresh.thread_data, sizeof(log->thread_data)); + + *fresh_info = (log_segment_info){ + .addr = log->addr, + .meta_addr = log->meta_head, + .magic = log->magic, + }; + return STATUS_OK; +} + +void +shard_log_segment_discard(cache *cc, const log_segment_info *segment) +{ + if (segment->meta_addr == 0) { + return; + } + refcount ref = mini_dec_ref(cc, segment->meta_addr, PAGE_TYPE_LOG); + platform_assert(ref == 0); +} + +void +shard_log_release(log_handle *logh) +{ + shard_log_zap((shard_log *)logh); +} + uint64 shard_log_addr(log_handle *logh) { @@ -374,7 +567,20 @@ shard_log_compare(const void *p1, const void *p2, void *unused) { log_entry **le1 = (log_entry **)p1; log_entry **le2 = (log_entry **)p2; - return (*le1)->generation - (*le2)->generation; + + if ((*le1)->memtable_generation < (*le2)->memtable_generation) { + return -1; + } + if ((*le1)->memtable_generation > (*le2)->memtable_generation) { + return 1; + } + if ((*le1)->leaf_generation < (*le2)->leaf_generation) { + return -1; + } + if ((*le1)->leaf_generation > (*le2)->leaf_generation) { + return 1; + } + return 0; } log_handle * @@ -527,6 +733,16 @@ shard_log_iterator_curr(iterator *itorh, key *curr_key, message *msg) *msg = log_entry_message(itor->cc, itor->entries[itor->pos]); } +void +shard_log_iterator_curr_generations(shard_log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation) +{ + platform_assert(itor->pos < itor->num_entries); + *memtable_generation = itor->entries[itor->pos]->memtable_generation; + *leaf_generation = itor->entries[itor->pos]->leaf_generation; +} + bool32 shard_log_iterator_can_prev(iterator *itorh) { @@ -597,11 +813,12 @@ shard_log_print(shard_log *log) le = log_entry_next(le)) { platform_default_log( - "%s -- %s%s : %lu\n", + "%s -- %s%s : memtable=%lu leaf=%lu\n", key_string(dcfg, log_entry_key(le)), log_entry_message_is_blob(le) ? "(blob) " : "", message_string(dcfg, log_entry_message(cc, le)), - le->generation); + le->memtable_generation, + le->leaf_generation); } } cache_unget(cc, page); diff --git a/src/shard_log.h b/src/shard_log.h index 6459d78d..086381a7 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -46,6 +46,8 @@ typedef struct shard_log { uint64 addr; uint64 meta_head; uint64 magic; + /* Set once any log page has been allocated; survives sealing. */ + bool32 has_pages; } shard_log; typedef struct log_entry log_entry; @@ -76,9 +78,22 @@ typedef struct ONDISK shard_log_hdr { platform_status shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg); +platform_status +shard_log_rotate(log_handle *log, + log_segment_info *sealed, + log_segment_info *fresh); + void shard_log_zap(shard_log *log); +/* + * Drop the external mini-allocator ownership transferred to a detached + * segment. It is for failed rotation cleanup and tests; persisted segment + * descriptors will eventually own and release this reference instead. + */ +void +shard_log_segment_discard(cache *cc, const log_segment_info *segment); + platform_status shard_log_iterator_init(cache *cc, shard_log_config *cfg, @@ -90,6 +105,15 @@ shard_log_iterator_init(cache *cc, void shard_log_iterator_deinit(platform_heap_id hid, shard_log_iterator *itor); +/* + * Return the generation metadata of the current record. The caller must + * first establish that the iterator has a current record. + */ +void +shard_log_iterator_curr_generations(shard_log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation); + void shard_log_config_init(shard_log_config *log_cfg, cache_config *cache_cfg, diff --git a/src/splinterdb.c b/src/splinterdb.c index b8614668..c40dc8c3 100644 --- a/src/splinterdb.c +++ b/src/splinterdb.c @@ -498,7 +498,8 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN deinit_cache: clockcache_deinit(&kvs->cache_handle); deinit_allocator: - rc_allocator_unmount(&kvs->allocator_handle); + /* Initialization/open failed before a successful clean core shutdown. */ + rc_allocator_deinit(&kvs->allocator_handle); deinit_system: task_system_deinit(&kvs->task_sys); deinit_iohandle: @@ -578,7 +579,13 @@ splinterdb_close(splinterdb **kvs_in) // IN } io_wait_all(kvs->io_handle); clockcache_deinit(&kvs->cache_handle); - rc_allocator_unmount(&kvs->allocator_handle); + if (SUCCESS(status)) { + /* This writes the map and only then publishes allocator clean state. */ + rc_allocator_unmount(&kvs->allocator_handle); + } else { + /* Keep the durable clean marker cleared; the next open must rebuild. */ + rc_allocator_deinit(&kvs->allocator_handle); + } task_system_deinit(&kvs->task_sys); io_handle_destroy(kvs->io_handle); diff --git a/src/trunk.c b/src/trunk.c index 133139a5..4ea8bdb9 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -2008,7 +2008,6 @@ trunk_node_serialize_maybe_setup_next_page(cache *cc, "%s():%d: cache_alloc() failed", __func__, __LINE__); return STATUS_NO_MEMORY; } - cache_mark_dirty(cc, *current_page); *page_offset = 0; } @@ -2131,7 +2130,6 @@ trunk_node_serialize(trunk_context *context, trunk_node *node) rc = STATUS_NO_MEMORY; goto cleanup; } - cache_mark_dirty(context->cc, header_page); int64 min_inflight_bundle_start = trunk_node_first_live_inflight_bundle(node); @@ -2541,6 +2539,65 @@ trunk_read_end(trunk_context *context) batch_rwlock_unget(&context->root_lock, 0); } +/* + * Capture the root address and acquire its allocator reference while the root + * lock prevents COW publication from dropping the live reference. This is + * deliberately metadata-only: checkpointing must not fault the root page just + * to take a snapshot. + */ +platform_status +trunk_snapshot_create(trunk_context *context, trunk_snapshot *snapshot) +{ + snapshot->root_addr = 0; + + trunk_read_begin(context); + if (context->root != NULL) { + snapshot->root_addr = context->root->ref.addr; + trunk_addr_inc_ref(context->al, snapshot->root_addr); + } + trunk_read_end(context); + + return STATUS_OK; +} + +platform_status +trunk_snapshot_create_from_addr(allocator *al, + uint64 root_addr, + trunk_snapshot *snapshot) +{ + snapshot->root_addr = root_addr; + if (root_addr != 0) { + trunk_addr_inc_ref(al, root_addr); + } + return STATUS_OK; +} + +platform_status +trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot) +{ + if (snapshot->root_addr == 0) { + return STATUS_OK; + } + + /* + * Route the release through a scratch context so the ordinary COW + * teardown path (trunk_context_deinit -> trunk_ondisk_node_ref_destroy) + * performs the single decrement snapshot's reference is owed. The scratch + * context only needs context's shared cfg/cc/al/ts; it does not need to + * share context's root. + */ + trunk_context scratch; + platform_status rc = trunk_context_init( + &scratch, context->cfg, context->hid, context->cc, context->al, + context->ts, *snapshot); + snapshot->root_addr = 0; // trunk_context_init consumes it regardless of rc + if (!SUCCESS(rc)) { + return rc; + } + trunk_context_deinit(&scratch); + return STATUS_OK; +} + platform_status trunk_init_root_handle(trunk_context *context, trunk_ondisk_node_handle *handle) { @@ -6705,7 +6762,7 @@ trunk_context_init(trunk_context *context, cache *cc, allocator *al, task_system *ts, - uint64 root_addr) + trunk_snapshot snapshot) { memset(context, 0, sizeof(trunk_context)); @@ -6715,12 +6772,20 @@ trunk_context_init(trunk_context *context, context->al = al; context->ts = ts; - if (root_addr != 0) { + if (snapshot.root_addr != 0) { + // Adopt: snapshot already carries an owned reference (from + // trunk_snapshot_create or trunk_snapshot_create_from_addr), so this + // does not take a new one; it just gives that reference a home. context->root = trunk_ondisk_node_ref_create( - context, NEGATIVE_INFINITY_KEY, root_addr, FALSE); + context, NEGATIVE_INFINITY_KEY, snapshot.root_addr, TRUE /* adopt */); if (context->root == NULL) { platform_error_log("trunk_node_context_init: " "ondisk_node_ref_create failed\n"); + // The reference was never actually adopted (creation failed before + // that could happen), but the caller has already relinquished it. + // Discharge it here so this function always consumes its snapshot, + // regardless of outcome. + trunk_ondisk_node_dec_ref(context, snapshot.root_addr); return STATUS_NO_MEMORY; } } @@ -6746,34 +6811,6 @@ trunk_context_init(trunk_context *context, return STATUS_OK; } -void -trunk_inc_ref(allocator *al, uint64 root_addr) -{ - trunk_addr_inc_ref(al, root_addr); -} - -platform_status -trunk_dec_ref(const trunk_config *cfg, - platform_heap_id hid, - cache *cc, - allocator *al, - task_system *ts, - uint64 root_addr) -{ - trunk_context context; - platform_status rc = - trunk_context_init(&context, cfg, hid, cc, al, ts, root_addr); - if (!SUCCESS(rc)) { - platform_error_log("trunk_node_dec_ref: trunk_node_context_init failed: " - "%d\n", - rc.r); - return rc; - } - trunk_ondisk_node_dec_ref(&context, root_addr); - trunk_context_deinit(&context); - return STATUS_OK; -} - void trunk_context_deinit(trunk_context *context) { @@ -6790,34 +6827,6 @@ trunk_context_deinit(trunk_context *context) } } - -platform_status -trunk_context_clone(trunk_context *dst, trunk_context *src) -{ - platform_status rc; - trunk_ondisk_node_handle handle; - rc = trunk_init_root_handle(src, &handle); - if (!SUCCESS(rc)) { - platform_error_log("trunk_node_context_clone: trunk_init_root_handle " - "failed: %d\n", - rc.r); - return rc; - } - uint64 root_addr = handle.header_page->disk_addr; - - rc = trunk_context_init( - dst, src->cfg, src->hid, src->cc, src->al, src->ts, root_addr); - trunk_ondisk_node_handle_deinit(&handle); - return rc; -} - -platform_status -trunk_make_durable(trunk_context *context) -{ - cache_flush(context->cc); - return STATUS_OK; -} - /************************************ * Stats ************************************/ diff --git a/src/trunk.h b/src/trunk.h index 7df20d09..70786bec 100644 --- a/src/trunk.h +++ b/src/trunk.h @@ -174,6 +174,15 @@ typedef struct trunk_context { incorporation_tasks tasks; } trunk_context; +/* + * An owned, point-in-time reference to a COW trunk root. The reference keeps + * the root (and therefore its reachable subtree) alive until it is either + * released or transferred to a durable checkpoint record. + */ +typedef struct trunk_snapshot { + uint64 root_addr; +} trunk_snapshot; + typedef struct trunk_ondisk_node_handle { cache *cc; page_handle *header_page; @@ -196,6 +205,14 @@ trunk_config_init(trunk_config *config, uint64 prefetch_budget, bool32 use_stats); +/* + * Initializes context with a root of snapshot. Consumes snapshot's owned + * reference regardless of outcome: on success the reference now lives in + * context (released by a later trunk_context_deinit); on failure it has + * already been discharged. Either way, the caller must not use or release + * snapshot again. A null snapshot (root_addr == 0) starts an empty trunk with + * no root, as for a freshly formatted table. + */ platform_status trunk_context_init(trunk_context *context, const trunk_config *cfg, @@ -203,37 +220,35 @@ trunk_context_init(trunk_context *context, cache *cc, allocator *al, task_system *ts, - uint64 root_addr); + trunk_snapshot snapshot); void -trunk_inc_ref(allocator *al, uint64 root_addr); +trunk_context_deinit(trunk_context *context); +/* Capture an owned reference to the current COW root without reading it. */ platform_status -trunk_dec_ref(const trunk_config *cfg, - platform_heap_id hid, - cache *cc, - allocator *al, - task_system *ts, - uint64 root_addr); +trunk_snapshot_create(trunk_context *context, trunk_snapshot *snapshot); -void -trunk_context_deinit(trunk_context *context); - -/* Create a writable snapshot of a trunk */ +/* + * Capture an owned reference to a root_addr that is not (or not yet) any live + * context's root -- e.g. one just read out of a durable checkpoint record. + * root_addr must already be referenced by the caller (its persisted refcount + * accounts for it); this takes out an additional, independent reference. A + * null root_addr (0) produces a null snapshot without touching the allocator. + */ platform_status -trunk_context_clone(trunk_context *dst, trunk_context *src); +trunk_snapshot_create_from_addr(allocator *al, + uint64 root_addr, + trunk_snapshot *snapshot); -/* Make a trunk durable */ +/* Drop an owned snapshot reference that was not published. */ platform_status -trunk_make_durable(trunk_context *context); +trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot); /******************************** * Mutations ********************************/ -void -trunk_modification_begin(trunk_context *context); - // Build a new trunk with the branch incorporated. The new trunk is not yet // visible to queriers. platform_status @@ -259,8 +274,6 @@ trunk_optimize(trunk_context *context, bool32 full_leaf_compactions, struct splinterdb_notification *notification); -void -trunk_modification_end(trunk_context *context); /******************************** * Queries @@ -270,9 +283,6 @@ platform_status trunk_init_root_handle(trunk_context *context, trunk_ondisk_node_handle *handle); -uint64 -trunk_ondisk_node_handle_addr(const trunk_ondisk_node_handle *handle); - void trunk_ondisk_node_handle_deinit(trunk_ondisk_node_handle *handle); diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 2511e615..560e4c24 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -17,6 +17,7 @@ #include "rc_allocator.h" #include "cache.h" #include "clockcache.h" +#include "mini_allocator.h" #include "random.h" #include "task.h" @@ -70,6 +71,97 @@ test_btree_process_noop(void *arg, uint64 generation) // really a no-op } +static platform_status +test_memtable_generation_init(cache *cc, + test_btree_config *cfg, + platform_heap_id hid) +{ + const uint64 first_generation = 17; + const uint64 max_memtables = 4; + memtable_config mt_cfg = *cfg->mt_cfg; + memtable_context mt_ctxt; + + mt_cfg.max_memtables = max_memtables; + + platform_status rc = memtable_context_init( + &mt_ctxt, hid, cc, &mt_cfg, test_btree_process_noop, NULL); + if (!SUCCESS(rc)) { + return rc; + } + + if (mt_ctxt.generation != 0 || mt_ctxt.generation_to_incorporate != 0 + || mt_ctxt.generation_retired != (uint64)-1) + { + platform_error_log("memtable fresh initialization has unexpected " + "generation state\n"); + rc = STATUS_TEST_FAILED; + goto deinit_fresh; + } + + for (uint64 generation = 0; generation < max_memtables; generation++) { + uint64 mt_no = generation % max_memtables; + if (mt_ctxt.mt[mt_no].generation != generation) { + platform_error_log("memtable fresh initialization put generation " + "%lu in the wrong slot\n", + generation); + rc = STATUS_TEST_FAILED; + goto deinit_fresh; + } + } + +deinit_fresh: + memtable_context_deinit(&mt_ctxt); + if (!SUCCESS(rc)) { + return rc; + } + + rc = memtable_context_init_at_generation(&mt_ctxt, + hid, + cc, + &mt_cfg, + test_btree_process_noop, + NULL, + first_generation); + if (!SUCCESS(rc)) { + return rc; + } + + if (mt_ctxt.generation != first_generation + || mt_ctxt.generation_to_incorporate != first_generation + || mt_ctxt.generation_retired != first_generation - 1) + { + platform_error_log("memtable recovery initialization has unexpected " + "generation state\n"); + rc = STATUS_TEST_FAILED; + goto deinit_recovery; + } + + for (uint64 generation = first_generation; + generation < first_generation + max_memtables; + generation++) + { + uint64 mt_no = generation % max_memtables; + if (mt_ctxt.mt[mt_no].generation != generation) { + platform_error_log("memtable recovery initialization put generation " + "%lu in the wrong slot\n", + generation); + rc = STATUS_TEST_FAILED; + goto deinit_recovery; + } + } + + memtable_block_inserts(&mt_ctxt); + memtable_unblock_inserts(&mt_ctxt); + +deinit_recovery: + memtable_context_deinit(&mt_ctxt); + if (SUCCESS(rc)) { + platform_default_log( + "btree_test: memtable generation init test passed\n"); + } + return rc; +} + test_memtable_context * test_memtable_context_create(cache *cc, test_btree_config *cfg, @@ -2258,6 +2350,9 @@ btree_test(int argc, char *argv[]) platform_assert_status_ok(rc); cache *ccp = (cache *)cc; + rc = test_memtable_generation_init(ccp, &test_cfg, hid); + platform_assert_status_ok(rc); + uint64 max_tuples_per_memtable = test_cfg.mt_cfg->max_extents_per_memtable * cache_config_extent_size((cache_config *)&system_cfg.cache_cfg) / 3 diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index ad4b3b27..6dd55db3 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -12,6 +12,7 @@ #include "rc_allocator.h" #include "cache.h" #include "clockcache.h" +#include "mini_allocator.h" #include "task.h" #include "random.h" #include "test_common.h" @@ -90,6 +91,997 @@ cache_test_alloc_extents(cache *cc, return rc; } +static platform_status +cache_test_fill_page(cache *cc, uint64 addr, uint64 page_size, uint8 value) +{ + page_handle *page = cache_get(cc, addr, TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, page)) { + cache_unget(cc, page); + return STATUS_BUSY; + } + + cache_lock(cc, page); + memset(page->data, value, page_size); + cache_unlock(cc, page); + cache_unclaim(cc, page); + cache_unget(cc, page); + return STATUS_OK; +} + +static platform_status +cache_test_verify_disk_page(cache *cc, + uint64 addr, + uint64 page_size, + uint8 expected) +{ + clockcache *clock = (clockcache *)cc; + buffer_handle buffer; + platform_status rc = platform_buffer_init(&buffer, page_size); + if (!SUCCESS(rc)) { + return rc; + } + + rc = io_read(clock->io, platform_buffer_getaddr(&buffer), page_size, addr); + if (SUCCESS(rc)) { + const uint8 *bytes = platform_buffer_getaddr(&buffer); + for (uint64 i = 0; i < page_size; i++) { + if (bytes[i] != expected) { + platform_error_log("cache_test: unexpected byte at addr=%lu " + "offset=%lu: got=%u expected=%u\n", + addr, + i, + bytes[i], + expected); + rc = STATUS_TEST_FAILED; + break; + } + } + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +/* + * Corrupt one full raw-I/O page and make that corruption durable. The + * allocator clean-state records are intentionally outside the cache, so this + * lets the bootstrap test exercise A/B fallback without test-only allocator + * hooks. + */ +static platform_status +cache_test_zero_disk_page(io_handle *io, uint64 addr, uint64 page_size) +{ + buffer_handle buffer; + platform_status rc = platform_buffer_init(&buffer, page_size); + if (!SUCCESS(rc)) { + return rc; + } + + void *page = platform_buffer_getaddr(&buffer); + memset(page, 0, page_size); + rc = io_write(io, page, page_size, addr); + if (SUCCESS(rc)) { + rc = io_durable_barrier(io); + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +/* + * The recovery allocator must bootstrap only the extents it owns itself, then + * let recovery walkers rebuild all other ownership. In particular, it must + * not trust a refcount table persisted by an earlier clean shutdown. + */ +static platform_status +test_rc_allocator_recovery_bootstrap(allocator_config *cfg, + io_handle *io, + platform_heap_id hid) +{ + const allocator_root_id root_id = 1; + uint64 refcount_buffer_size = + ROUNDUP(cfg->extent_capacity * sizeof(refcount), cfg->io_cfg->page_size); + uint64 refcount_extent_count = + (refcount_buffer_size + cfg->io_cfg->extent_size - 1) + / cfg->io_cfg->extent_size; + uint64 reserved_extent_count = 1 + refcount_extent_count + 2; + uint64 old_clean_state_addr = + (1 + refcount_extent_count + 1) * cfg->io_cfg->extent_size; + uint64 super_addr = 0; + uint64 stale_extent_addr = 0; + platform_status rc = STATUS_OK; + rc_allocator original, recovery, remounted; + bool32 original_live = FALSE; + bool32 recovery_live = FALSE; + bool32 remounted_live = FALSE; + + ZERO_CONTENTS(&original); + ZERO_CONTENTS(&recovery); + ZERO_CONTENTS(&remounted); + platform_default_log( + "cache_test: allocator recovery bootstrap test started\n"); + + rc = rc_allocator_init(&original, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + original_live = TRUE; + + rc = + allocator_alloc_super_addr((allocator *)&original, root_id, &super_addr); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = allocator_alloc( + (allocator *)&original, &stale_extent_addr, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* Persist a non-reserved extent that the recovery mount must ignore. */ + rc_allocator_unmount(&original); + original_live = FALSE; + + /* + * New allocator initialization writes false records with sequences 1 and + * 2. The first clean close writes the true record into slot 0 (sequence + * 3), making slot 1 the older false record. Destroying that older slot + * must not prevent a normal mount from trusting the surviving true slot. + */ + rc = cache_test_zero_disk_page( + io, old_clean_state_addr, cfg->io_cfg->page_size); + if (!SUCCESS(rc)) { + goto cleanup; + } + + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + platform_error_log("cache_test: normal mount did not fall back to its " + "surviving clean-state record\n"); + goto cleanup; + } + remounted_live = TRUE; + if (allocator_get_refcount((allocator *)&remounted, stale_extent_addr) + != AL_ONE_REF) + { + platform_error_log("cache_test: normal mount did not load the clean " + "refcount map\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + /* Simulate a crash after normal mount durably marked the allocator dirty. */ + rc_allocator_deinit(&remounted); + remounted_live = FALSE; + + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); + if (rc.r != STATUS_INVALID_STATE.r) { + platform_error_log("cache_test: normal mount trusted an allocator after " + "a simulated mounted crash\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = STATUS_OK; + + rc = rc_allocator_mount_recovery( + &recovery, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_live = TRUE; + + uint64 mounted_super_addr = 0; + rc = allocator_get_super_addr( + (allocator *)&recovery, root_id, &mounted_super_addr); + if (!SUCCESS(rc) || mounted_super_addr != super_addr) { + platform_error_log("cache_test: recovery lost a persisted superblock " + "mapping\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + if (allocator_in_use((allocator *)&recovery) != reserved_extent_count) { + platform_error_log("cache_test: recovery expected %lu reserved extents, " + "found %lu\n", + reserved_extent_count, + allocator_in_use((allocator *)&recovery)); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) { + uint64 extent_addr = extent_no * cfg->io_cfg->extent_size; + if (allocator_get_refcount((allocator *)&recovery, extent_addr) + != AL_ONE_REF) + { + platform_error_log("cache_test: recovery did not reserve extent %lu\n", + extent_no); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + } + if (allocator_get_refcount((allocator *)&recovery, stale_extent_addr) + != AL_FREE) + { + platform_error_log("cache_test: recovery reused a persisted data " + "refcount\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + rc = rc_allocator_rebuild_acquire_extent( + &recovery, stale_extent_addr, PAGE_TYPE_MISC); + if (!SUCCESS(rc) + || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) + != AL_ONE_REF) + { + platform_error_log("cache_test: first rebuilt extent reference was " + "incorrect\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + /* + * Abort never persists a partial rebuild and leaves the allocator marked + * unclean, so a normal mount must refuse to trust the old refcount map. + */ + rc_allocator_abort_recovery(&recovery); + recovery_live = FALSE; + + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); + if (rc.r != STATUS_INVALID_STATE.r) { + platform_error_log("cache_test: normal mount trusted an unclean " + "allocator map\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = STATUS_OK; + + rc = rc_allocator_mount_recovery( + &recovery, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_live = TRUE; + + rc = rc_allocator_rebuild_acquire_extent( + &recovery, stale_extent_addr, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = rc_allocator_rebuild_acquire_extent( + &recovery, stale_extent_addr, PAGE_TYPE_MISC); + if (!SUCCESS(rc) + || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) + != AL_ONE_REF + 1) + { + platform_error_log("cache_test: rebuilt extent reference did not " + "increment\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc_allocator_rebuild_finish(&recovery); + rc_allocator_unmount(&recovery); + recovery_live = FALSE; + + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + remounted_live = TRUE; + if (allocator_get_refcount((allocator *)&remounted, stale_extent_addr) + != AL_ONE_REF + 1) + { + platform_error_log("cache_test: finished recovery was not persisted by " + "clean unmount\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + +cleanup: + if (remounted_live) { + rc_allocator_deinit(&remounted); + } + if (recovery_live) { + if (recovery.recovery_in_progress) { + rc_allocator_abort_recovery(&recovery); + } else { + rc_allocator_deinit(&recovery); + } + } + if (original_live) { + rc_allocator_deinit(&original); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: allocator recovery bootstrap test " + "passed\n"); + } + return rc; +} + +#define TEST_MINI_RECOVER_NUM_DATA_EXTENTS 3 + +/* + * Build a real mini_allocator with one metadata extent and several data + * extents, durably persist it, then simulate a crash: mount a fresh allocator + * in recovery mode (which deliberately ignores the persisted refcount table) + * and confirm mini_recover_allocations() rediscovers every extent from the + * on-disk metadata alone -- including the ordering property this whole + * mechanism exists for, since a rebuilding allocator considers all of these + * extents unallocated until mini_recover_allocations() itself establishes + * their references, ahead of the cache_get() calls it issues to read them. + */ +static platform_status +test_mini_recover_allocations(allocator_config *allocator_cfg, + clockcache_config *cache_cfg, + io_handle *io, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + rc_allocator original, recovery; + clockcache original_cc, recovery_cc; + bool32 original_al_live = FALSE, original_cc_live = FALSE; + bool32 recovery_al_live = FALSE, recovery_cc_live = FALSE; + mini_allocator mini; + bool32 mini_live = FALSE; + uint64 meta_head = 0; + uint64 data_extents[TEST_MINI_RECOVER_NUM_DATA_EXTENTS] = {0}; + + platform_default_log( + "cache_test: mini_recover_allocations test started\n"); + + rc = rc_allocator_init( + &original, allocator_cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + original_al_live = TRUE; + + rc = clockcache_init(&original_cc, + cache_cfg, + io, + (allocator *)&original, + "test", + hid, + platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + original_cc_live = TRUE; + cache *original_ccp = (cache *)&original_cc; + + rc = allocator_alloc((allocator *)&original, &meta_head, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + goto cleanup; + } + mini_init(&mini, original_ccp, meta_head, 0, 1, PAGE_TYPE_MISC); + mini_live = TRUE; + + for (uint64 i = 0; i < TEST_MINI_RECOVER_NUM_DATA_EXTENTS; i++) { + data_extents[i] = mini_alloc_extent(&mini, 0, NULL); + if (data_extents[i] == 0) { + rc = STATUS_NO_SPACE; + goto cleanup; + } + } + mini_release(&mini); + mini_live = FALSE; + + /* Durably persist before "crashing": recovery must see real disk state. */ + cache_flush(original_ccp); + clockcache_deinit(&original_cc); + original_cc_live = FALSE; + rc_allocator_unmount(&original); + original_al_live = FALSE; + + rc = rc_allocator_mount_recovery( + &recovery, allocator_cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_al_live = TRUE; + + /* mount_recovery deliberately ignores the persisted map: confirm that. */ + if (allocator_get_refcount((allocator *)&recovery, meta_head) != AL_FREE) { + platform_error_log( + "cache_test: recovery mount trusted a persisted refcount\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + rc = clockcache_init(&recovery_cc, + cache_cfg, + io, + (allocator *)&recovery, + "test", + hid, + platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_cc_live = TRUE; + + /* + * Every extent below is still AL_FREE from recovery's point of view: this + * call must establish each one's reference itself, before its own + * cache_get() calls, or those cache_get()s would trip the cache's debug + * allocation check. + */ + rc = mini_recover_allocations( + (cache *)&recovery_cc, meta_head, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + platform_error_log("cache_test: mini_recover_allocations failed: %s\n", + platform_status_to_string(rc)); + goto cleanup; + } + + /* + * Exactly one reference per extent is mini_recover_allocations()'s own + * documented contract; reproducing a mini_allocator's full live-lifetime + * refcount (which may include self-references beyond this) is explicitly + * the job of a higher-level recovery walker, not this function. + */ + if (allocator_get_refcount((allocator *)&recovery, meta_head) + != AL_ONE_REF) + { + platform_error_log( + "cache_test: mini_recover_allocations did not recover the metadata " + "extent reference\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + for (uint64 i = 0; i < TEST_MINI_RECOVER_NUM_DATA_EXTENTS; i++) { + if (allocator_get_refcount((allocator *)&recovery, data_extents[i]) + != AL_ONE_REF) + { + platform_error_log("cache_test: mini_recover_allocations did not " + "recover data extent %lu\n", + i); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + } + rc_allocator_rebuild_finish(&recovery); + +cleanup: + if (mini_live) { + mini_release(&mini); + } + if (recovery_cc_live) { + clockcache_deinit(&recovery_cc); + } + if (recovery_al_live) { + if (recovery.recovery_in_progress) { + rc_allocator_abort_recovery(&recovery); + } else { + rc_allocator_deinit(&recovery); + } + } + if (original_cc_live) { + clockcache_deinit(&original_cc); + } + if (original_al_live) { + rc_allocator_deinit(&original); + } + + if (SUCCESS(rc)) { + platform_default_log( + "cache_test: mini_recover_allocations test passed\n"); + } + return rc; +} + +typedef struct { + cache *cc; + volatile bool32 finished; + platform_status status; +} cache_fence_test_context; + +static void +cache_test_writeback_fence_thread(void *arg) +{ + cache_fence_test_context *ctxt = (cache_fence_test_context *)arg; + ctxt->status = cache_writeback_dirty(ctxt->cc); + __atomic_store_n(&ctxt->finished, TRUE, __ATOMIC_RELEASE); +} + +/* + * Verify the simplified writeback-fence contract: a single pass flushes every + * dirty, unlocked page and completes even while another page is write-locked, + * skipping the locked page rather than waiting for it. old_page is left dirty + * and unlocked so the fence must flush it; new_page is held write-locked so the + * fence must skip it and still finish promptly. + */ +static platform_status +test_cache_writeback_fence(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *old_page = NULL; + page_handle *new_page = NULL; + bool32 new_claimed = FALSE, new_locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 old_baseline = 0x11, old_value = 0x22; + const uint8 new_baseline = 0x33, new_value = 0x44; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread fence_thread; + + platform_default_log("cache_test: writeback fence test started\n"); + platform_assert(cfg->page_capacity >= 2 * pages_per_extent); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, 2 * pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 2); + if (!SUCCESS(rc)) { + platform_free(hid, addr_arr); + addr_arr = NULL; + goto cleanup; + } + + rc = cache_test_fill_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), old_baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_fill_page(cc, + addr_arr[pages_per_extent], + cache_config_page_size(&cfg->super), + new_baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + /* Dirty old_page and leave it unlocked: the fence must flush it. */ + old_page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, old_page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + cache_lock(cc, old_page); + memset(old_page->data, old_value, cache_config_page_size(&cfg->super)); + cache_unlock(cc, old_page); + cache_unclaim(cc, old_page); + cache_unget(cc, old_page); + old_page = NULL; + + /* Hold new_page write-locked: the fence must skip it and still complete. */ + new_page = cache_get(cc, addr_arr[pages_per_extent], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, new_page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + new_claimed = TRUE; + cache_lock(cc, new_page); + new_locked = TRUE; + memset(new_page->data, new_value, cache_config_page_size(&cfg->super)); + + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_thread_started = TRUE; + + /* The fence must finish without waiting for the write-locked new_page. */ + timestamp wait_start = platform_get_timestamp(); + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { + platform_error_log( + "cache_test: fence did not complete while a page was locked\n"); + rc = STATUS_TIMEDOUT; + goto cleanup; + } + platform_sleep_ns(1000); + } + +cleanup: + if (new_locked) { + cache_unlock(cc, new_page); + } + if (new_claimed) { + cache_unclaim(cc, new_page); + } + if (new_page != NULL) { + cache_unget(cc, new_page); + } + if (old_page != NULL) { + cache_unget(cc, old_page); + } + if (fence_thread_started) { + platform_status join_rc = platform_thread_join(&fence_thread); + if (!SUCCESS(join_rc) && SUCCESS(rc)) { + rc = join_rc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + /* Only new_page, write-locked throughout the fence, remains dirty. */ + if (SUCCESS(rc)) { + uint32 dirty_count = cache_count_dirty(cc); + if (dirty_count != 1) { + platform_error_log("cache_test: expected one dirty page after fence, " + "found %u\n", + dirty_count); + rc = STATUS_TEST_FAILED; + } + } + if (SUCCESS(rc)) { + rc = cache_durable_barrier(cc); + } + /* The dirty, unlocked page was flushed by the fence. */ + if (SUCCESS(rc)) { + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), old_value); + } + /* The write-locked page was skipped, so its baseline is still on disk. */ + if (SUCCESS(rc)) { + rc = cache_test_verify_disk_page(cc, + addr_arr[pages_per_extent], + cache_config_page_size(&cfg->super), + new_baseline); + } + + if (addr_arr != NULL) { + cache_flush(cc); + for (uint32 i = 0; i < 2; i++) { + uint64 addr = addr_arr[i * pages_per_extent]; + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr, PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + } + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: writeback fence test passed\n"); + } else { + platform_default_log("cache_test: writeback fence test failed\n"); + } + return rc; +} + +/* + * A page that is only claimed (not write-locked) is not cleanable, so a fence + * skips it and completes; the page's baseline is therefore still what is on + * disk. Once the claim is dropped, a second fence flushes the page. + */ +static platform_status +test_cache_writeback_fence_claimed_page(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *page = NULL; + bool32 claimed = FALSE, locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 baseline = 0x55, final_value = 0x66; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread fence_thread; + + platform_default_log("cache_test: claimed-page fence test started\n"); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 1); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_fill_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + /* Dirty the page, then keep only a claim (release the write lock). */ + page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + claimed = TRUE; + cache_lock(cc, page); + locked = TRUE; + memset(page->data, final_value, cache_config_page_size(&cfg->super)); + cache_unlock(cc, page); + locked = FALSE; + + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_thread_started = TRUE; + + /* A claimed page is not cleanable, so the fence skips it and completes. */ + timestamp wait_start = platform_get_timestamp(); + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { + platform_error_log( + "cache_test: fence did not complete while page was claimed\n"); + rc = STATUS_TIMEDOUT; + goto cleanup; + } + platform_sleep_ns(1000); + } + + platform_status join_rc = platform_thread_join(&fence_thread); + fence_thread_started = FALSE; + if (!SUCCESS(join_rc)) { + rc = join_rc; + goto cleanup; + } + if (!SUCCESS(fence_ctxt.status)) { + rc = fence_ctxt.status; + goto cleanup; + } + + /* The skipped page was not written, so its baseline is still on disk. */ + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* Drop the claim; the page is now cleanable and a second fence flushes it. + */ + cache_unclaim(cc, page); + claimed = FALSE; + cache_unget(cc, page); + page = NULL; + + rc = cache_writeback_dirty(cc); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (cache_count_dirty(cc) != 0) { + platform_error_log("cache_test: second fence left dirty pages\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = cache_durable_barrier(cc); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), final_value); + +cleanup: + if (locked) { + cache_unlock(cc, page); + } + if (claimed) { + cache_unclaim(cc, page); + } + if (page != NULL) { + cache_unget(cc, page); + } + if (fence_thread_started) { + platform_status jrc = platform_thread_join(&fence_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + if (addr_arr != NULL) { + cache_flush(cc); + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr_arr[0], PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: claimed-page fence test passed\n"); + } else { + platform_default_log("cache_test: claimed-page fence test failed\n"); + } + return rc; +} + +typedef struct { + cache *cc; + uint64 *addrs; + uint64 num_addrs; + uint64 page_size; + volatile bool32 stop; +} cache_hammer_context; + +/* + * Continuously dirty and asynchronously write back a rotating set of pages, so + * the I/O subsystem never reaches global quiescence. + */ +static void +cache_test_hammer_thread(void *arg) +{ + cache_hammer_context *ctxt = (cache_hammer_context *)arg; + uint64 i = 0; + while (!__atomic_load_n(&ctxt->stop, __ATOMIC_ACQUIRE)) { + uint64 addr = ctxt->addrs[i % ctxt->num_addrs]; + page_handle *page = cache_get(ctxt->cc, addr, TRUE, PAGE_TYPE_MISC); + if (cache_try_claim(ctxt->cc, page)) { + cache_lock(ctxt->cc, page); + memset(page->data, (uint8)i, ctxt->page_size); + cache_unlock(ctxt->cc, page); + cache_unclaim(ctxt->cc, page); + cache_page_writeback(ctxt->cc, page, FALSE, PAGE_TYPE_MISC); + } + cache_unget(ctxt->cc, page); + i++; + } +} + +/* + * A writeback fence must drain only the pages dirty at its cut and then return, + * even while another thread keeps dirtying and writing back other pages. The + * previous io_wait_all implementation waited for *global* I/O quiescence, which + * this workload never reaches, so it would hang here. + */ +static platform_status +test_cache_writeback_fence_liveness(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 page_size = cache_config_page_size(&cfg->super); + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + const uint8 baseline = 0x11, target_value = 0x77; + bool32 hammer_started = FALSE, fence_started = FALSE; + cache_hammer_context hammer_ctxt = {.cc = cc, .stop = FALSE}; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread hammer_thread, fence_thread; + + platform_default_log("cache_test: writeback fence liveness test started\n"); + platform_assert(cfg->page_capacity >= 2 * pages_per_extent); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, 2 * pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 2); + if (!SUCCESS(rc)) { + platform_free(hid, addr_arr); + addr_arr = NULL; + goto cleanup; + } + rc = cache_test_fill_page(cc, addr_arr[0], page_size, baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + /* Dirty the target page (unlocked); the fence must flush it. */ + rc = cache_test_fill_page(cc, addr_arr[0], page_size, target_value); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* Hammer the second extent's pages to keep I/O perpetually in flight. */ + hammer_ctxt.addrs = &addr_arr[pages_per_extent]; + hammer_ctxt.num_addrs = pages_per_extent; + hammer_ctxt.page_size = page_size; + rc = platform_thread_create( + &hammer_thread, FALSE, cache_test_hammer_thread, &hammer_ctxt, hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + hammer_started = TRUE; + + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_started = TRUE; + + /* The fence must return despite the ongoing hammer activity. */ + timestamp wait_start = platform_get_timestamp(); + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(10)) { + platform_error_log( + "cache_test: fence did not terminate under concurrent writes\n"); + rc = STATUS_TIMEDOUT; + break; + } + platform_sleep_ns(1000); + } + +cleanup: + /* Stop the hammer first so I/O can quiesce even if the fence is stuck. */ + if (hammer_started) { + __atomic_store_n(&hammer_ctxt.stop, TRUE, __ATOMIC_RELEASE); + platform_status jrc = platform_thread_join(&hammer_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; + } + } + if (fence_started) { + platform_status jrc = platform_thread_join(&fence_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + /* The target page, dirty at the cut, must have been made durable. */ + if (SUCCESS(rc)) { + rc = cache_durable_barrier(cc); + } + if (SUCCESS(rc)) { + rc = + cache_test_verify_disk_page(cc, addr_arr[0], page_size, target_value); + } + + if (addr_arr != NULL) { + cache_flush(cc); + for (uint32 i = 0; i < 2; i++) { + uint64 addr = addr_arr[i * pages_per_extent]; + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr, PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + } + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log( + "cache_test: writeback fence liveness test passed\n"); + } else { + platform_default_log( + "cache_test: writeback fence liveness test failed\n"); + } + return rc; +} + platform_status test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) { @@ -98,6 +1090,21 @@ test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) page_handle **page_arr = NULL; uint64 *addr_arr = NULL; + rc = test_cache_writeback_fence(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + + rc = test_cache_writeback_fence_claimed_page(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + + rc = test_cache_writeback_fence_liveness(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + /* allocate twice as many pages as the cache capacity */ uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); uint32 extent_capacity = cfg->page_capacity / pages_per_extent; @@ -232,7 +1239,6 @@ test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) } } for (i = 0; i < cfg->page_capacity; i++) { - cache_mark_dirty(cc, page_arr[i]); cache_unlock(cc, page_arr[i]); cache_unclaim(cc, page_arr[i]); cache_unget(cc, page_arr[i]); @@ -418,7 +1424,6 @@ cache_test_dirty_flush(cache *cc, rc = STATUS_TEST_FAILED; break; } - cache_mark_dirty(cc, ph); cache_unlock(cc, ph); cache_unclaim(cc, ph); cache_unget(cc, ph); @@ -977,6 +1982,18 @@ cache_test(int argc, char *argv[]) goto cleanup; } + rc = + test_rc_allocator_recovery_bootstrap(&system_cfg.allocator_cfg, io, hid); + if (!SUCCESS(rc)) { + goto destroy_iohandle; + } + + rc = test_mini_recover_allocations( + &system_cfg.allocator_cfg, &system_cfg.cache_cfg, io, hid); + if (!SUCCESS(rc)) { + goto destroy_iohandle; + } + rc = test_init_task_system(&ts, hid, &system_cfg.task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n", diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index 1bf96ba5..fa17d8b4 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -19,6 +19,8 @@ #include "poison.h" +#define LOG_TEST_LEAVES_PER_MEMTABLE 97 + int test_log_crash(clockcache *cc, clockcache_config *cache_cfg, @@ -59,12 +61,44 @@ test_log_crash(clockcache *cc, merge_accumulator_init(&msg, hid); for (i = 0; i < num_entries; i++) { - key skey = - test_key(&keybuffer, TEST_RANDOM, i, 0, 0, 1 + (i % key_size), 0); - generate_test_message(gen, i, &msg); - log_write(logh, skey, merge_accumulator_to_message(&msg), i); + uint64 entry_num = i; + if (2 * LOG_TEST_LEAVES_PER_MEMTABLE <= num_entries + && i < 2 * LOG_TEST_LEAVES_PER_MEMTABLE) + { + /* + * Write the first two generations in the opposite order, and each + * generation in reverse leaf order. The iterator must restore the + * (memtable_generation, leaf_generation) order below. + */ + uint64 memtable_generation = 1 - i / LOG_TEST_LEAVES_PER_MEMTABLE; + uint64 leaf_generation = + LOG_TEST_LEAVES_PER_MEMTABLE - 1 - i % LOG_TEST_LEAVES_PER_MEMTABLE; + entry_num = memtable_generation * LOG_TEST_LEAVES_PER_MEMTABLE + + leaf_generation; + } + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + int log_rc = log_write(logh, + skey, + merge_accumulator_to_message(&msg), + entry_num / LOG_TEST_LEAVES_PER_MEMTABLE, + entry_num % LOG_TEST_LEAVES_PER_MEMTABLE); + platform_assert(log_rc == 0); } + rc = log_seal(logh); + platform_assert_status_ok(rc); + rc = cache_writeback_dirty((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + if (crash) { clockcache_deinit(cc); rc = clockcache_init( @@ -82,6 +116,12 @@ test_log_crash(clockcache *cc, generate_test_message(gen, i, &msg); message mmessage = merge_accumulator_to_message(&msg); iterator_curr(itorh, &returned_key, &returned_message); + uint64 memtable_generation; + uint64 leaf_generation; + shard_log_iterator_curr_generations( + &itor, &memtable_generation, &leaf_generation); + platform_assert(memtable_generation == i / LOG_TEST_LEAVES_PER_MEMTABLE); + platform_assert(leaf_generation == i % LOG_TEST_LEAVES_PER_MEMTABLE); if (data_key_compare(cfg->data_cfg, skey, returned_key) || message_lex_cmp(mmessage, returned_message)) { @@ -99,6 +139,8 @@ test_log_crash(clockcache *cc, } platform_default_log("log returned %lu of %lu entries\n", i, num_entries); + platform_assert(i == num_entries); + platform_assert(!iterator_can_curr(itorh)); merge_accumulator_deinit(&msg); @@ -108,6 +150,161 @@ test_log_crash(clockcache *cc, return 0; } +static void +test_log_write_range(log_handle *logh, + test_message_generator *gen, + platform_heap_id hid, + uint64 key_size, + uint64 first_entry, + uint64 num_entries) +{ + merge_accumulator msg; + DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); + merge_accumulator_init(&msg, hid); + + for (uint64 i = 0; i < num_entries; i++) { + uint64 entry_num = first_entry + i; + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + int log_rc = log_write( + logh, skey, merge_accumulator_to_message(&msg), entry_num, 0); + platform_assert(log_rc == 0); + } + + merge_accumulator_deinit(&msg); +} + +static void +test_log_verify_segment(cache *cc, + shard_log_config *cfg, + const log_segment_info *segment, + test_message_generator *gen, + platform_heap_id hid, + uint64 key_size, + uint64 first_entry, + uint64 num_entries) +{ + shard_log_iterator itor; + iterator *itorh = (iterator *)&itor; + merge_accumulator msg; + DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); + key returned_key; + message returned_message; + + platform_assert(segment->addr != 0); + platform_assert(segment->meta_addr != 0); + platform_status rc = shard_log_iterator_init( + cc, cfg, hid, segment->addr, segment->magic, &itor); + platform_assert_status_ok(rc); + + merge_accumulator_init(&msg, hid); + for (uint64 i = 0; i < num_entries; i++) { + uint64 entry_num = first_entry + i; + platform_assert(iterator_can_curr(itorh)); + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + iterator_curr(itorh, &returned_key, &returned_message); + uint64 memtable_generation; + uint64 leaf_generation; + shard_log_iterator_curr_generations( + &itor, &memtable_generation, &leaf_generation); + platform_assert(memtable_generation == entry_num); + platform_assert(leaf_generation == 0); + platform_assert(data_key_compare(cfg->data_cfg, skey, returned_key) == 0); + platform_assert( + message_lex_cmp(merge_accumulator_to_message(&msg), returned_message) + == 0); + rc = iterator_next(itorh); + platform_assert_status_ok(rc); + } + platform_assert(!iterator_can_curr(itorh)); + + merge_accumulator_deinit(&msg); + shard_log_iterator_deinit(hid, &itor); +} + +/* + * A rotation must permanently detach the first mini-allocator stream and + * produce a fresh one. Reinitializing the cache after each forced physical + * persistence cut makes this test exercise only persisted pages for both + * identities; it does not model logical durable-tail publication. + */ +static int +test_log_rotate(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + shard_log *log, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + const uint64 old_first = 1000, old_count = 16; + const uint64 new_first = 2000, new_count = 16; + log_segment_info sealed, fresh; + + platform_status rc = shard_log_init(log, (cache *)cc, cfg); + platform_assert_status_ok(rc); + + test_log_write_range( + (log_handle *)log, gen, hid, key_size, old_first, old_count); + rc = log_rotate((log_handle *)log, &sealed, &fresh); + platform_assert_status_ok(rc); + platform_assert(sealed.addr != 0); + platform_assert(sealed.meta_addr != 0); + platform_assert(fresh.addr != 0); + platform_assert(fresh.meta_addr != 0); + platform_assert(sealed.meta_addr != fresh.meta_addr); + platform_assert(sealed.magic != fresh.magic); + + rc = cache_writeback_dirty((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + + clockcache_deinit(cc); + rc = clockcache_init( + cc, cache_cfg, io, al, "rotated-old", hid, platform_get_module_id()); + platform_assert_status_ok(rc); + test_log_verify_segment( + (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); + + test_log_write_range( + (log_handle *)log, gen, hid, key_size, new_first, new_count); + rc = log_seal((log_handle *)log); + platform_assert_status_ok(rc); + rc = cache_writeback_dirty((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + + clockcache_deinit(cc); + rc = clockcache_init( + cc, cache_cfg, io, al, "rotated-new", hid, platform_get_module_id()); + platform_assert_status_ok(rc); + test_log_verify_segment( + (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); + test_log_verify_segment( + (cache *)cc, cfg, &fresh, gen, hid, key_size, new_first, new_count); + + shard_log_segment_discard((cache *)cc, &sealed); + shard_log_zap(log); + return 0; +} + static int test_log_large_message(cache *cc, shard_log_config *cfg, @@ -133,8 +330,8 @@ test_log_large_message(cache *cc, merge_accumulator_set_class(&msg, MESSAGE_TYPE_INSERT); memset(merge_accumulator_data(&msg), 'L', value_len); - int log_rc = - log_write((log_handle *)log, skey, merge_accumulator_to_message(&msg), 0); + int log_rc = log_write( + (log_handle *)log, skey, merge_accumulator_to_message(&msg), 0, 0); platform_assert(log_rc == 0); merge_accumulator filler; @@ -145,12 +342,22 @@ test_log_large_message(cache *cc, memset( merge_accumulator_data(&filler), 'f', merge_accumulator_length(&filler)); for (uint64 i = 1; i < 16; i++) { - log_rc = log_write( - (log_handle *)log, skey, merge_accumulator_to_message(&filler), i); + log_rc = log_write((log_handle *)log, + skey, + merge_accumulator_to_message(&filler), + i / 4, + i % 4); platform_assert(log_rc == 0); } merge_accumulator_deinit(&filler); + rc = log_seal((log_handle *)log); + platform_assert_status_ok(rc); + rc = cache_writeback_dirty(cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier(cc); + platform_assert_status_ok(rc); + rc = shard_log_iterator_init(cc, cfg, hid, @@ -201,7 +408,9 @@ test_log_thread(void *arg) for (i = thread_id * num_entries; i < (thread_id + 1) * num_entries; i++) { key skey = test_key(&keybuf, TEST_RANDOM, i, 0, 0, key_size, 0); generate_test_message(gen, i, &msg); - log_write(logh, skey, merge_accumulator_to_message(&msg), i); + int log_rc = log_write( + logh, skey, merge_accumulator_to_message(&msg), i / 1024, i % 1024); + platform_assert(log_rc == 0); } merge_accumulator_deinit(&msg); @@ -380,6 +589,17 @@ log_test(int argc, char *argv[]) rc = test_log_large_message((cache *)cc, &system_cfg.log_cfg, log, hid); platform_assert(rc == 0); + rc = test_log_rotate(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + log, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); + if (run_perf_test) { ret = test_log_perf((cache *)cc, &system_cfg.log_cfg, diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 5377cde9..01fe15f6 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -250,6 +250,79 @@ CTEST2(splinter, test_inserts) core_destroy(&spl); } +/* + * The second checkpoint slot is a torn-write fallback, not permission for a + * normal mount to silently roll back past a newer, valid active record. A + * successful mount publishes such an active record; until crash recovery is + * implemented, a concurrent/restarted normal mount must reject it even + * though the preceding clean record remains valid in the other slot. + */ +CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) +{ + allocator *alp = (allocator *)&data->al; + allocator_root_id root_id = test_generate_allocator_root_id(); + core_handle created, mounted, rejected, cleanup; + platform_status rc; + + rc = core_mkfs(&created, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + /* Give the clean record a real COW root, not just the empty-tree root. */ + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + test_key(&keybuf, TEST_RANDOM, 1, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 1, &msg); + rc = core_insert(&created, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + merge_accumulator_deinit(&msg); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_unmount(&created); + ASSERT_TRUE(SUCCESS(rc)); + + /* This mount advances the A/B sequence with an unmounted=FALSE record. */ + rc = core_mount(&mounted, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_mount(&rejected, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); + + /* Finish cleanly, then prove the same record pair is mountable again. */ + rc = core_unmount(&mounted); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_mount(&cleanup, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + core_destroy(&cleanup); +} + static void trunk_shadow_init(trunk_shadow *shadow, data_config *data_cfg,