From 2c21f8b27130920e09cbddc717cf06b5a43c1f87 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 12:22:56 -0700 Subject: [PATCH 01/13] initial attempt Signed-off-by: Rob Johnson --- src/cache.h | 53 +- src/clockcache.c | 427 ++++++++++++++-- src/clockcache.h | 15 + src/core.c | 842 +++++++++++++++++++++++++------ src/core.h | 4 + src/log.h | 68 ++- src/memtable.c | 92 ++-- src/memtable.h | 29 ++ src/mini_allocator.c | 241 +++++++++ src/mini_allocator.h | 51 ++ src/platform_linux/laio.c | 30 ++ src/platform_linux/platform_io.h | 17 + src/rc_allocator.c | 574 +++++++++++++++++++-- src/rc_allocator.h | 61 ++- src/shard_log.c | 243 ++++++++- src/shard_log.h | 25 + src/splinterdb.c | 11 +- src/trunk.c | 47 +- src/trunk.h | 17 + tests/functional/btree_test.c | 199 ++++++++ tests/functional/cache_test.c | 698 +++++++++++++++++++++++++ tests/functional/log_test.c | 260 +++++++++- tests/unit/splinter_test.c | 79 +++ 23 files changed, 3779 insertions(+), 304 deletions(-) diff --git a/src/cache.h b/src/cache.h index bc239b7a0..52bec1434 100644 --- a/src/cache.h +++ b/src/cache.h @@ -97,6 +97,8 @@ cache_config_extent_page(const cache_config *cfg, uint64 extent_addr, uint64 i) typedef void (*cache_generic_fn)(cache *cc); typedef uint64 (*cache_generic_uint64_fn)(cache *cc); typedef void (*page_generic_fn)(cache *cc, page_handle *page); +typedef platform_status (*cache_durable_barrier_fn)(cache *cc); +typedef platform_status (*cache_writeback_fence_fn)(cache *cc); typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); @@ -176,6 +178,8 @@ typedef struct cache_ops { page_sync_fn page_sync; extent_sync_fn extent_sync; cache_generic_fn flush; + cache_writeback_fence_fn writeback_fence; + cache_durable_barrier_fn durable_barrier; evict_fn evict; cache_generic_fn cleanup; page_addr_pred_fn in_use; @@ -366,8 +370,10 @@ cache_unclaim(cache *cc, page_handle *page) * * Blocks until outstanding read locks are released by other threads. * - * If you call this method, you almost certainly want to call - * cache_mark_dirty() immediately afterward. + * Clockcache conservatively begins a dirty interval on this transition, so a + * checkpoint fence cannot miss a caller that makes its first change before + * calling cache_mark_dirty(). cache_mark_dirty() remains the explicit, + * idempotent declaration of intent to modify the page. *---------------------------------------------------------------------- */ static inline void @@ -435,11 +441,10 @@ cache_prefetch_page(cache *cc, uint64 addr, page_type type) * The caller had better have the write lock on the page via cache_lock() * before changing its value. * - * TODO This method should be removed; its effect should come automatically - * with the acquisition of a write lock. @robj reports lots of bugs - * due to forgetting to call this method. And we can't think of a case - * where we'd want the "optimization" of taking a write lock but then - * decide not to dirty it. + * Clockcache already begins a dirty interval when the caller acquires the + * write lock; this call is therefore idempotent there. It remains part of the + * cache API to document the mutation and preserve the contract for other + * implementations. *---------------------------------------------------------------------- */ static inline void @@ -542,6 +547,40 @@ cache_flush(cache *cc) cc->ops->flush(cc); } +/* + *----------------------------------------------------------------------------- + * cache_writeback_fence + * + * Wait until every page whose current dirty interval began before this call's + * cut has completed writeback. Pages dirtied after the cut do not delay the + * call. The cache implementation must inspect resident metadata only; it must + * not fault pages in merely to satisfy the fence. + * + * This is the checkpointing primitive. It is intentionally narrower than + * cache_flush(), which attempts to leave the entire cache clean. + *----------------------------------------------------------------------------- + */ +static inline platform_status +cache_writeback_fence(cache *cc) +{ + return cc->ops->writeback_fence(cc); +} + +/* + *----------------------------------------------------------------------------- + * cache_durable_barrier + * + * Ensure that writeback completed before this call is durable across a power + * loss. Callers normally use this after cache_writeback_fence(), and again + * after publishing a checkpoint superblock. + *----------------------------------------------------------------------------- + */ +static inline platform_status +cache_durable_barrier(cache *cc) +{ + return cc->ops->durable_barrier(cc); +} + /* *----------------------------------------------------------------------------- * cache_evict diff --git a/src/clockcache.c b/src/clockcache.c index b8197236e..74aba087b 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -141,6 +141,9 @@ clockcache_print(platform_log_handle *log_handle, clockcache *cc); #define CC_LOADING (1u << 4) // page is actively being read from disk #define CC_WRITELOCKED (1u << 5) // write lock is held #define CC_CLAIMED (1u << 6) // claim is held +// A checkpoint fence has selected this dirty interval for writeback. New +// writers must not claim the page until that writeback completes. +#define CC_WRITEBACK_REQUESTED (1u << 7) /* Common status flag combinations */ // free entry @@ -222,6 +225,70 @@ clockcache_test_flag(clockcache *cc, uint32 entry_number, entry_status flag) return flag & clockcache_get_status(cc, entry_number); } +/* + *-------------------------------------------------------------------------- + * Dirty-generation bookkeeping + * + * A dirty generation identifies the clean->dirty interval, rather than every + * individual mutation. A checkpoint fence rotates the current generation and + * drains all older intervals. Writers cannot modify a page during writeback, + * so writing a later version of an older dirty interval still satisfies the + * fence. + *-------------------------------------------------------------------------- + */ +static void +clockcache_dirty_begin(clockcache *cc, uint32 entry_number) +{ + platform_status rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + debug_assert(entry->dirty_generation == 0); + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + clockcache_clear_flag(cc, entry_number, CC_CLEAN); + } + + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); +} + +static void +clockcache_dirty_complete_writeback(clockcache *cc, uint32 entry_number) +{ + platform_status rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + debug_assert(entry->dirty_generation != 0); + entry->dirty_generation = 0; + debug_only uint32 was_clean = + clockcache_set_flag(cc, entry_number, CC_CLEAN); + debug_assert(!was_clean); + debug_only uint32 was_writeback = + clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); + debug_assert(was_writeback); + clockcache_clear_flag(cc, entry_number, CC_WRITEBACK_REQUESTED); + + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); +} + +static void +clockcache_dirty_discard(clockcache *cc, uint32 entry_number) +{ + platform_status rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + entry->dirty_generation = 0; + clockcache_clear_flag(cc, entry_number, CC_WRITEBACK_REQUESTED); + + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); +} + #ifdef RECORD_ACQUISITION_STACKS static void clockcache_record_backtrace(clockcache *cc, uint32 entry_number) @@ -627,11 +694,27 @@ clockcache_try_get_claim(clockcache *cc, uint32 entry_number) entry_number, clockcache_test_flag(cc, entry_number, CC_CLAIMED)); + if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK_REQUESTED)) { + return GET_RC_CONFLICT; + } + if (clockcache_set_flag(cc, entry_number, CC_CLAIMED)) { clockcache_log(0, entry_number, "return false\n", NULL); return GET_RC_CONFLICT; } + /* + * A fence may have selected the dirty interval between the first test and + * the claim. Do not let a new writer keep that selected interval dirty; + * drop the claim and let the requested writeback proceed instead. + */ + if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK_REQUESTED)) { + debug_only uint32 was_claimed = + clockcache_clear_flag(cc, entry_number, CC_CLAIMED); + debug_assert(was_claimed); + return GET_RC_CONFLICT; + } + clockcache_record_backtrace(cc, entry_number); return GET_RC_SUCCESS; @@ -782,11 +865,16 @@ clockcache_try_get_write(clockcache *cc, uint32 entry_number) static inline bool32 clockcache_ok_to_writeback(clockcache *cc, uint32 entry_number, - bool32 with_access) + bool32 with_access, + bool32 requested_only) { uint32 status = clockcache_get_status(cc, entry_number); - return ((status == CC_CLEANABLE1_STATUS) - || (with_access && status == CC_CLEANABLE2_STATUS)); + uint32 request_flag = status & CC_WRITEBACK_REQUESTED; + uint32 base_status = status & ~CC_WRITEBACK_REQUESTED; + + return ((!requested_only || request_flag) + && ((base_status == CC_CLEANABLE1_STATUS) + || (with_access && base_status == CC_CLEANABLE2_STATUS))); } /* @@ -812,14 +900,24 @@ clockcache_try_set_writeback(clockcache *cc, volatile uint32 *status = &cc->entry[entry_number].status; if (__sync_bool_compare_and_swap( - status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) + status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS) + || __sync_bool_compare_and_swap(status, + CC_CLEANABLE1_STATUS + | CC_WRITEBACK_REQUESTED, + CC_WRITEBACK1_STATUS + | CC_WRITEBACK_REQUESTED)) { return TRUE; } if (with_access - && __sync_bool_compare_and_swap( - status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) + && (__sync_bool_compare_and_swap( + status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS) + || __sync_bool_compare_and_swap(status, + CC_CLEANABLE2_STATUS + | CC_WRITEBACK_REQUESTED, + CC_WRITEBACK2_STATUS + | CC_WRITEBACK_REQUESTED))) { return TRUE; } @@ -861,8 +959,6 @@ clockcache_write_callback(void *wbs) uint32 entry_number; clockcache_entry *entry; uint64 addr; - debug_only uint32 debug_status; - for (i = 0; i < count; i++) { entry_number = clockcache_data_to_entry_number(cc, (char *)iovec[i].iov_base); @@ -876,10 +972,7 @@ clockcache_write_callback(void *wbs) entry_number, addr); - debug_status = clockcache_set_flag(cc, entry_number, CC_CLEAN); - debug_assert(!debug_status); - debug_status = clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); - debug_assert(debug_status); + clockcache_dirty_complete_writeback(cc, entry_number); } if (state->outstanding_pages) { @@ -922,8 +1015,11 @@ clockcache_abort_writeback_range(clockcache *cc, * they are not. *---------------------------------------------------------------------- */ -void -clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) +static platform_status +clockcache_batch_start_writeback(clockcache *cc, + uint64 batch, + bool32 is_urgent, + bool32 requested_only) { uint32 entry_no, next_entry_no; uint64 addr, first_addr, end_addr, i; @@ -932,6 +1028,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) uint64 end_entry_no = start_entry_no + CC_ENTRIES_PER_BATCH; clockcache_entry *entry, *next_entry; + platform_status result = STATUS_OK; debug_assert((tid < MAX_THREADS), "Invalid tid=%lu\n", tid); debug_assert(cc != NULL); @@ -953,7 +1050,8 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) entry = &cc->entry[entry_no]; addr = entry->page.disk_addr; // test and test and set in the if condition - if (clockcache_ok_to_writeback(cc, entry_no, is_urgent) + if (clockcache_ok_to_writeback( + cc, entry_no, is_urgent, requested_only) && clockcache_try_set_writeback(cc, entry_no, is_urgent)) { debug_assert(clockcache_lookup(cc, addr) == entry_no); @@ -968,6 +1066,8 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY + && clockcache_ok_to_writeback( + cc, next_entry_no, is_urgent, requested_only) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); first_addr += page_size; end_addr = entry->page.disk_addr; @@ -981,6 +1081,8 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY + && clockcache_ok_to_writeback( + cc, next_entry_no, is_urgent, requested_only) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); @@ -991,6 +1093,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) "clockcache_batch_start_writeback: async_io_state allocation " "failed\n"); clockcache_abort_writeback_range(cc, first_addr, end_addr); + result = STATUS_NO_MEMORY; goto close_log; } @@ -1008,6 +1111,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) platform_status_to_string(rc)); clockcache_abort_writeback_range(cc, first_addr, end_addr); platform_free(PROCESS_PRIVATE_HEAP_ID, state); + result = rc; goto close_log; } @@ -1033,6 +1137,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) io_async_state_deinit(state->iostate); clockcache_abort_writeback_range(cc, first_addr, end_addr); platform_free(PROCESS_PRIVATE_HEAP_ID, state); + result = rc; goto close_log; } } @@ -1042,12 +1147,156 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) cc->stats[tid].writes_issued++; } - io_async_run(state->iostate); + if (io_async_run(state->iostate) == ASYNC_STATUS_DONE) { + rc = io_async_state_get_result(state->iostate); + if (SUCCESS(rc)) { + platform_error_log( + "clockcache_batch_start_writeback: async write for addr " + "%lu completed without invoking its callback\n", + first_addr); + rc = STATUS_IO_ERROR; + } + clockcache_abort_writeback_range(cc, first_addr, end_addr); + io_async_state_deinit(state->iostate); + platform_free(PROCESS_PRIVATE_HEAP_ID, state); + result = rc; + goto close_log; + } } } close_log: clockcache_close_log_stream(); + return result; +} + +/* + *-------------------------------------------------------------------------- + * clockcache_fence_request_old_dirty -- + * + * Mark every dirty interval at or before cutoff for writeback. The dirty lock + * makes this atomic with both a clean->dirty transition and a writeback + * completion, so a false return means no interval from this fence remains. + *-------------------------------------------------------------------------- + */ +static bool32 +clockcache_fence_request_old_dirty(clockcache *cc, uint64 cutoff) +{ + bool32 pending = FALSE; + platform_status rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_no); + if (entry->dirty_generation != 0 + && entry->dirty_generation <= cutoff) + { + clockcache_set_flag(cc, entry_no, CC_WRITEBACK_REQUESTED); + pending = TRUE; + } + } + + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); + return pending; +} + +/* + *-------------------------------------------------------------------------- + * clockcache_fence_cancel_requests -- + * + * Abandon a failed fence without leaving writers permanently blocked behind + * its request bits. A later fence will select these still-dirty generations + * again. + *-------------------------------------------------------------------------- + */ +static void +clockcache_fence_cancel_requests(clockcache *cc, uint64 cutoff) +{ + platform_status rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_no); + if (entry->dirty_generation != 0 + && entry->dirty_generation <= cutoff) + { + clockcache_clear_flag(cc, entry_no, CC_WRITEBACK_REQUESTED); + } + } + + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); +} + +/* + *-------------------------------------------------------------------------- + * clockcache_fence_start_writeback_round -- + * + * Start writeback only for intervals selected by a checkpoint fence. This is + * a metadata scan of resident cache entries; no page lookup or trunk walk is + * involved. The normal clock hand may retain batch_busy while it owns a free + * batch, so a fence intentionally does not wait for that batch-level state. + * Per-entry compare-and-swap on CC_WRITEBACK provides the required exclusion + * from the normal cleaner. + *-------------------------------------------------------------------------- + */ +static platform_status +clockcache_fence_start_writeback_round(clockcache *cc) +{ + for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { + platform_status rc = + clockcache_batch_start_writeback(cc, batch, TRUE, TRUE); + if (!SUCCESS(rc)) { + return rc; + } + } + return STATUS_OK; +} + +/* + *-------------------------------------------------------------------------- + * clockcache_writeback_fence -- + * + * Rotate the dirty generation, then wait only for dirty intervals that were + * already present at that cut. Pages dirtied after the cut carry the next + * generation and do not delay this checkpoint. A request bit blocks new + * writers from indefinitely extending a selected interval, while a writer + * that was already active is allowed to finish and its final bytes are what + * gets written. + *-------------------------------------------------------------------------- + */ +platform_status +clockcache_writeback_fence(clockcache *cc) +{ + platform_status rc = platform_mutex_lock(&cc->writeback_fence_lock); + platform_assert_status_ok(rc); + + rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + uint64 cutoff = __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + platform_assert(cutoff < UINT64_MAX); + __atomic_store_n(&cc->dirty_generation, cutoff + 1, __ATOMIC_RELEASE); + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + while (clockcache_fence_request_old_dirty(cc, cutoff)) { + rc = clockcache_fence_start_writeback_round(cc); + if (!SUCCESS(rc)) { + clockcache_fence_cancel_requests(cc, cutoff); + break; + } + /* + * Poll the local completion queue, then yield briefly if all selected + * pages are currently write-locked or in another cleaner's I/O. + */ + clockcache_wait(cc); + platform_sleep_ns(1000); + } + + platform_status unlock_rc = platform_mutex_unlock(&cc->writeback_fence_lock); + platform_assert_status_ok(unlock_rc); + return rc; } /* @@ -1148,6 +1397,7 @@ clockcache_try_evict(clockcache *cc, uint32 entry_number) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); entry->type = PAGE_TYPE_INVALID; + clockcache_dirty_discard(cc, entry_number); entry->status = CC_FREE_STATUS; clockcache_log( addr, entry_number, "evict: entry %u addr %lu\n", entry_number, addr); @@ -1228,7 +1478,12 @@ clockcache_move_hand(clockcache *cc, bool32 is_urgent) cleaner_hand = (evict_hand + cc->cleaner_gap) % cc->cfg->batch_capacity; clean_batch_busy = &cc->batch_busy[cleaner_hand]; if (__sync_bool_compare_and_swap(clean_batch_busy, FALSE, TRUE)) { - clockcache_batch_start_writeback(cc, cleaner_hand, is_urgent); + platform_status rc = clockcache_batch_start_writeback( + cc, cleaner_hand, is_urgent, FALSE); + if (!SUCCESS(rc)) { + platform_error_log("clockcache_move_hand: writeback failed: %s\n", + platform_status_to_string(rc)); + } was_busy = __sync_bool_compare_and_swap(clean_batch_busy, TRUE, FALSE); debug_assert(was_busy); } @@ -1251,7 +1506,8 @@ clockcache_get_free_page(clockcache *cc, uint32 status, page_type type, bool32 refcount, - bool32 blocking) + bool32 blocking, + bool32 begins_dirty) { uint32 entry_no; uint64 num_passes = 0; @@ -1261,6 +1517,7 @@ clockcache_get_free_page(clockcache *cc, timestamp wait_start; debug_assert((tid < MAX_THREADS), "Invalid tid=%lu\n", tid); + debug_assert(!begins_dirty || status == CC_ALLOC_STATUS); if (cc->per_thread[tid].free_hand == CC_UNMAPPED_ENTRY) { clockcache_move_hand(cc, FALSE); } @@ -1275,16 +1532,45 @@ clockcache_get_free_page(clockcache *cc, uint64 end_entry = start_entry + CC_ENTRIES_PER_BATCH; for (entry_no = start_entry; entry_no < end_entry; entry_no++) { entry = &cc->entry[entry_no]; - if (entry->status == CC_FREE_STATUS - && __sync_bool_compare_and_swap( - &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS)) - { + if (entry->status == CC_FREE_STATUS) { + platform_status rc = STATUS_OK; + if (begins_dirty) { + rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + } + + bool32 reserved = __sync_bool_compare_and_swap( + &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS); + if (!reserved) { + if (begins_dirty) { + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); + } + continue; + } + + /* + * The allocation's dirty interval is tagged while holding the + * same lock as a fence cut. This prevents a checkpoint from + * seeing a dirty CC_ALLOC_STATUS entry with generation zero. + */ + debug_assert(entry->dirty_generation == 0); + if (begins_dirty) { + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + } + entry->status = status; + entry->type = type; + + if (begins_dirty) { + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); + } + if (refcount) { clockcache_inc_ref(cc, entry_no, tid); } platform_assert(entry->waiters.head == NULL); - entry->status = status; - entry->type = type; debug_assert(entry->page.disk_addr == CC_UNMAPPED_ADDR); clockcache_record_backtrace(cc, entry_no); return entry_no; @@ -1343,7 +1629,9 @@ clockcache_flush(clockcache *cc) flush_hand < cc->cfg->page_capacity / CC_ENTRIES_PER_BATCH; flush_hand++) { - clockcache_batch_start_writeback(cc, flush_hand, TRUE); + platform_status rc = + clockcache_batch_start_writeback(cc, flush_hand, TRUE, FALSE); + platform_assert_status_ok(rc); } // make sure all aio is complete again @@ -1404,7 +1692,8 @@ clockcache_alloc(clockcache *cc, uint64 addr, page_type type) CC_ALLOC_STATUS, type, TRUE, // refcount - TRUE); // blocking + TRUE, // blocking + TRUE); // begins_dirty clockcache_entry *entry = &cc->entry[entry_no]; entry->page.disk_addr = addr; entry->type = type; @@ -1504,6 +1793,7 @@ clockcache_try_page_discard(clockcache *cc, uint64 addr) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); entry->type = PAGE_TYPE_INVALID; + clockcache_dirty_discard(cc, entry_number); entry->status = CC_FREE_STATUS; /* 7. reset pincount */ @@ -1638,7 +1928,8 @@ clockcache_acquire_entry_for_load(clockcache *cc, // IN CC_READ_LOADING_STATUS, type, TRUE, // refcount - TRUE); // blocking + TRUE, // blocking + FALSE); // begins_dirty clockcache_entry *entry = clockcache_get_entry(cc, entry_number); /* * If someone else is loading the page and has reserved the lookup, let them @@ -2166,6 +2457,7 @@ clockcache_lock(clockcache *cc, page_handle *page) entry_number, page->disk_addr); clockcache_get_write(cc, entry_number); + clockcache_dirty_begin(cc, entry_number); } void @@ -2202,7 +2494,7 @@ clockcache_mark_dirty(clockcache *cc, page_handle *page) "mark_dirty: entry %u addr %lu\n", entry_number, entry->page.disk_addr); - clockcache_clear_flag(cc, entry_number, CC_CLEAN); + clockcache_dirty_begin(cc, entry_number); return; } @@ -2266,9 +2558,26 @@ clockcache_page_sync(clockcache *cc, const threadid tid = platform_get_tid(); platform_status status; - if (!clockcache_try_set_writeback(cc, entry_number, TRUE)) { - platform_assert(clockcache_test_flag(cc, entry_number, CC_CLEAN)); - return; + while (!clockcache_try_set_writeback(cc, entry_number, TRUE)) { + if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { + return; + } + + /* + * A pressure cleaner or a checkpoint fence may have begun writeback + * after the caller released its claim. Wait for that writeback rather + * than treating a perfectly valid concurrent flush as an assertion. + */ + if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK)) { + clockcache_wait(cc); + continue; + } + + platform_assert(0, + "page_sync requires a cleanable page: entry=%u " + "status=%u\n", + entry_number, + clockcache_get_status(cc, entry_number)); } if (cc->cfg->use_stats) { @@ -2330,11 +2639,7 @@ clockcache_page_sync(clockcache *cc, "page_sync write entry %u addr %lu\n", entry_number, addr); - debug_only uint8 rc; - rc = clockcache_set_flag(cc, entry_number, CC_CLEAN); - debug_assert(!rc); - rc = clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); - debug_assert(rc); + clockcache_dirty_complete_writeback(cc, entry_number); } } @@ -2593,7 +2898,7 @@ clockcache_prefetch_pages(clockcache *cc, case GET_RC_EVICTED: { uint32 free_entry_no = clockcache_get_free_page( - cc, CC_READ_LOADING_STATUS, type, FALSE, TRUE); + cc, CC_READ_LOADING_STATUS, type, FALSE, TRUE, FALSE); clockcache_entry *entry = &cc->entry[free_entry_no]; entry->page.disk_addr = addr; entry->type = type; @@ -3171,6 +3476,20 @@ clockcache_flush_virtual(cache *c) clockcache_flush(cc); } +platform_status +clockcache_writeback_fence_virtual(cache *c) +{ + clockcache *cc = (clockcache *)c; + return clockcache_writeback_fence(cc); +} + +platform_status +clockcache_durable_barrier_virtual(cache *c) +{ + clockcache *cc = (clockcache *)c; + return io_durable_barrier(cc->io); +} + int clockcache_evict_all_virtual(cache *c, bool32 ignore_pinned) { @@ -3305,6 +3624,8 @@ static cache_ops clockcache_ops = { .page_sync = clockcache_page_sync_virtual, .extent_sync = clockcache_extent_sync_virtual, .flush = clockcache_flush_virtual, + .writeback_fence = clockcache_writeback_fence_virtual, + .durable_barrier = clockcache_durable_barrier_virtual, .evict = clockcache_evict_all_virtual, .cleanup = clockcache_wait_virtual, .in_use = clockcache_in_use_virtual, @@ -3397,9 +3718,28 @@ clockcache_init(clockcache *cc, // OUT cc->io = io; cc->heap_id = hid; + platform_status rc = platform_mutex_init(&cc->dirty_lock, mid, hid); + if (!SUCCESS(rc)) { + platform_error_log("clockcache_init: failed to initialize dirty lock: %s\n", + platform_status_to_string(rc)); + goto alloc_error; + } + + rc = platform_mutex_init(&cc->writeback_fence_lock, mid, hid); + if (!SUCCESS(rc)) { + platform_error_log( + "clockcache_init: failed to initialize writeback fence lock: %s\n", + platform_status_to_string(rc)); + platform_status destroy_rc = platform_mutex_destroy(&cc->dirty_lock); + platform_assert_status_ok(destroy_rc); + goto alloc_error; + } + cc->dirty_generation = 1; + cc->dirty_locks_initialized = TRUE; + /* lookup maps addrs to entries, entry contains the entries themselves */ - platform_status rc = platform_buffer_init( - &cc->lookup_bh, allocator_page_capacity * sizeof(cc->lookup[0])); + rc = platform_buffer_init(&cc->lookup_bh, + allocator_page_capacity * sizeof(cc->lookup[0])); if (!SUCCESS(rc)) { platform_error_log("clockcache_init: failed to allocate lookup table " "(%lu bytes): %s\n", @@ -3440,6 +3780,7 @@ clockcache_init(clockcache *cc, // OUT cc->data + clockcache_multiply_by_page_size(cc, i); cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; cc->entry[i].status = CC_FREE_STATUS; + cc->entry[i].dirty_generation = 0; cc->entry[i].type = PAGE_TYPE_INVALID; async_wait_queue_init(&cc->entry[i].waiters); } @@ -3522,6 +3863,14 @@ clockcache_deinit(clockcache *cc) // IN/OUT #endif } + if (cc->dirty_locks_initialized) { + rc = platform_mutex_destroy(&cc->writeback_fence_lock); + platform_assert_status_ok(rc); + rc = platform_mutex_destroy(&cc->dirty_lock); + platform_assert_status_ok(rc); + cc->dirty_locks_initialized = FALSE; + } + if (cc->lookup) { rc = platform_buffer_deinit(&cc->lookup_bh); if (!SUCCESS(rc)) { diff --git a/src/clockcache.h b/src/clockcache.h index 814cde6a0..1e601bd54 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -10,6 +10,7 @@ #pragma once #include "platform_buffer.h" +#include "platform_mutex.h" #include "platform_threads.h" #include "allocator.h" #include "cache.h" @@ -72,6 +73,9 @@ typedef uint32 entry_status; // Saved in clockcache_entry->status struct clockcache_entry { page_handle page; volatile entry_status status; + // Generation in which this page's current dirty interval began. Zero means + // no dirty interval (a clean page or an unpublished clean-load reservation). + volatile uint64 dirty_generation; page_type type; async_wait_queue waiters; #ifdef RECORD_ACQUISITION_STACKS @@ -139,6 +143,17 @@ struct clockcache { volatile bool32 *batch_busy; // Convenience pointer for batch_bh uint64 cleaner_gap; + /* + * Dirty-generation bookkeeping for checkpoint writeback fences. The dirty + * lock serializes the clean<->dirty transition with a fence cut and with + * writeback completion. Fence rounds are serialized separately so a page + * only needs one outstanding writeback-request bit. + */ + platform_mutex dirty_lock; + platform_mutex writeback_fence_lock; + uint64 dirty_generation; + bool32 dirty_locks_initialized; + volatile struct { volatile uint32 free_hand; bool32 enable_sync_get; diff --git a/src/core.c b/src/core.c index d499f24a9..3faac28ed 100644 --- a/src/core.c +++ b/src/core.c @@ -46,8 +46,38 @@ static const int64 latency_histo_buckets[LATENCYHISTO_SIZE] = { _Static_assert(CORE_NUM_MEMTABLES <= MAX_MEMTABLES, "CORE_NUM_MEMTABLES <= MAX_MEMTABLES"); -/* Some randomly chosen Splinter super-block checksum seed. */ -#define CORE_SUPER_CSUM_SEED (42) +/* Checkpoint metadata has independently checksummed directory and records. */ +#define CORE_CHECKPOINT_DIRECTORY_CSUM_SEED (42) +#define CORE_CHECKPOINT_RECORD_CSUM_SEED (43) + +#define CORE_CHECKPOINT_FORMAT_VERSION (2) +#define CORE_CHECKPOINT_RECORD_COUNT (2) + +#define CORE_CHECKPOINT_DIRECTORY_MAGIC (0x534442434B505444ULL) // SDBCKPTD +#define CORE_CHECKPOINT_RECORD_MAGIC (0x534442434B505452ULL) // SDBCKPTR + +static platform_status +core_checkpoint_lock_init(core_handle *spl) +{ + platform_status rc = platform_mutex_init(&spl->checkpoint_lock, + platform_get_module_id(), + spl->heap_id); + if (SUCCESS(rc)) { + spl->checkpoint_lock_initialized = TRUE; + } + return rc; +} + +static void +core_checkpoint_lock_deinit(core_handle *spl) +{ + if (!spl->checkpoint_lock_initialized) { + return; + } + platform_status rc = platform_mutex_destroy(&spl->checkpoint_lock); + platform_assert_status_ok(rc); + spl->checkpoint_lock_initialized = FALSE; +} /* * core logging functions. @@ -107,142 +137,542 @@ core_close_log_stream_if_enabled(core_handle *spl, /* *----------------------------------------------------------------------------- - * Splinter Super Block: Disk-resident structure. - * Super block lives on page of page type == PAGE_TYPE_SUPERBLOCK. + * Checkpoint metadata: disk-resident structures. + * + * allocator_get_super_addr() identifies a fixed page in the allocator's + * bootstrap extent. That page is an immutable directory, written exactly + * once when the table is created. The directory names two independently + * allocated record extents. Checkpoint publication alternates between their + * first pages, leaving one formerly valid record untouched if a new write is + * torn. + * + * This is intentionally a new on-disk format. Do not interpret a legacy + * core_super_block as a directory: doing so would turn arbitrary old fields + * into allocator-owned addresses. Existing databases must be migrated or + * reformatted before using this checkpoint metadata format. *----------------------------------------------------------------------------- */ -typedef struct ONDISK core_super_block { - uint64 root_addr; // Address of the root of the trunk for the instance - // referenced by this superblock. - uint64 log_addr; - uint64 log_meta_addr; - uint64 timestamp; - bool32 checkpointed; - bool32 unmounted; +typedef struct ONDISK core_checkpoint_directory { + uint64 magic; + uint64 format_version; + uint64 table_id; + uint64 record_addr[CORE_CHECKPOINT_RECORD_COUNT]; checksum128 checksum; -} core_super_block; +} core_checkpoint_directory; -/* - *----------------------------------------------------------------------------- - * Super block functions - *----------------------------------------------------------------------------- - */ -static platform_status -core_set_super_block(core_handle *spl, - bool32 is_checkpoint, - bool32 is_unmount, - bool32 is_create) +typedef struct ONDISK core_checkpoint_record { + /* + * The highest memtable generation incorporated in root_addr. The boolean + * keeps the fresh-database case distinct from generation zero. + */ + uint64 incorporated_generation; + bool32 has_incorporated_generation; + uint64 root_addr; + uint64 timestamp; + uint64 sequence; + uint64 table_id; + uint32 record_slot; + bool32 checkpointed; + bool32 unmounted; + uint64 magic; + uint64 format_version; + checksum128 checksum; +} core_checkpoint_record; + +typedef struct core_checkpoint_records { + core_checkpoint_record record[CORE_CHECKPOINT_RECORD_COUNT]; + bool32 valid[CORE_CHECKPOINT_RECORD_COUNT]; + bool32 have_newest; + uint64 newest_slot; + bool32 have_newest_unmounted; + uint64 newest_unmounted_slot; +} core_checkpoint_records; + +static checksum128 +core_checkpoint_directory_checksum(const core_checkpoint_directory *directory) { - uint64 super_addr; - page_handle *super_page; - core_super_block *super; - uint64 wait = 1; - platform_status rc; + return platform_checksum128(directory, + offsetof(core_checkpoint_directory, checksum), + CORE_CHECKPOINT_DIRECTORY_CSUM_SEED); +} - if (is_create) { - rc = allocator_alloc_super_addr(spl->al, spl->id, &super_addr); - } else { - rc = allocator_get_super_addr(spl->al, spl->id, &super_addr); +static checksum128 +core_checkpoint_record_checksum(const core_checkpoint_record *record) +{ + return platform_checksum128(record, + offsetof(core_checkpoint_record, checksum), + CORE_CHECKPOINT_RECORD_CSUM_SEED); +} + +static bool32 +core_checkpoint_record_addr_is_valid(core_handle *spl, uint64 addr) +{ + allocator_config *allocator_cfg = allocator_get_config(spl->al); + uint64 page_size = cache_page_size(spl->cc); + + return addr != 0 && addr % allocator_cfg->io_cfg->extent_size == 0 + && addr < allocator_cfg->capacity + && page_size <= allocator_cfg->capacity - addr; +} + +static bool32 +core_checkpoint_directory_is_valid(core_handle *spl, + const core_checkpoint_directory *directory) +{ + if (directory->magic != CORE_CHECKPOINT_DIRECTORY_MAGIC + || directory->format_version != CORE_CHECKPOINT_FORMAT_VERSION + || directory->table_id != spl->id + || !platform_checksum_is_equal( + directory->checksum, core_checkpoint_directory_checksum(directory))) + { + return FALSE; } + + uint64 record0 = directory->record_addr[0]; + uint64 record1 = directory->record_addr[1]; + allocator_config *allocator_cfg = allocator_get_config(spl->al); + return core_checkpoint_record_addr_is_valid(spl, record0) + && core_checkpoint_record_addr_is_valid(spl, record1) + && record0 != record1 + && !allocator_config_pages_share_extent(allocator_cfg, record0, record1); +} + +static bool32 +core_checkpoint_record_is_valid(core_handle *spl, + const core_checkpoint_record *record, + uint64 record_slot) +{ + return record->magic == CORE_CHECKPOINT_RECORD_MAGIC + && record->format_version == CORE_CHECKPOINT_FORMAT_VERSION + && record->table_id == spl->id && record->record_slot == record_slot + && record->sequence != 0 + && (record->has_incorporated_generation == FALSE + || record->has_incorporated_generation == TRUE) + && (record->has_incorporated_generation + ? record->incorporated_generation < UINT64_MAX + : record->incorporated_generation == 0) + && platform_checksum_is_equal( + record->checksum, core_checkpoint_record_checksum(record)); +} + +static void +core_write_checkpoint_page(core_handle *spl, + uint64 page_addr, + const void *contents, + uint64 contents_size) +{ + page_handle *page = + cache_get(spl->cc, page_addr, TRUE, PAGE_TYPE_SUPERBLOCK); + uint64 wait = 1; + while (!cache_try_claim(spl->cc, page)) { + cache_unget(spl->cc, page); + platform_sleep_ns(wait); + wait = wait > 1024 ? wait : 2 * wait; + page = cache_get(spl->cc, page_addr, TRUE, PAGE_TYPE_SUPERBLOCK); + } + cache_lock(spl->cc, page); + platform_assert(contents_size <= cache_page_size(spl->cc)); + memset(page->data, 0, cache_page_size(spl->cc)); + memcpy(page->data, contents, contents_size); + cache_mark_dirty(spl->cc, page); + cache_unlock(spl->cc, page); + cache_unclaim(spl->cc, page); + cache_page_sync(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); + cache_unget(spl->cc, page); +} + +static void +core_initialize_checkpoint_record_page(core_handle *spl, uint64 page_addr) +{ + page_handle *page = cache_alloc(spl->cc, page_addr, PAGE_TYPE_SUPERBLOCK); + platform_assert(page != NULL); + memset(page->data, 0, cache_page_size(spl->cc)); + cache_mark_dirty(spl->cc, page); + cache_unlock(spl->cc, page); + cache_unclaim(spl->cc, page); + cache_page_sync(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); + cache_unget(spl->cc, page); +} + +static platform_status +core_create_checkpoint_directory(core_handle *spl, + core_checkpoint_directory *directory) +{ + uint64 directory_addr; + platform_status rc = + allocator_alloc_super_addr(spl->al, spl->id, &directory_addr); if (!SUCCESS(rc)) { - platform_error_log("core_set_super_block: failed to %s super block " - "address for root id %lu: %s\n", - is_create ? "allocate" : "get", + platform_error_log("core_create_checkpoint_directory: failed to allocate " + "directory address for root id %lu: %s\n", spl->id, platform_status_to_string(rc)); return rc; } - super_page = cache_get(spl->cc, super_addr, TRUE, PAGE_TYPE_SUPERBLOCK); - while (!cache_try_claim(spl->cc, super_page)) { - platform_sleep_ns(wait); - wait *= 2; + + ZERO_CONTENTS(directory); + directory->magic = CORE_CHECKPOINT_DIRECTORY_MAGIC; + directory->format_version = CORE_CHECKPOINT_FORMAT_VERSION; + directory->table_id = spl->id; + + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + uint64 record_addr; + rc = allocator_alloc(spl->al, &record_addr, PAGE_TYPE_SUPERBLOCK); + if (!SUCCESS(rc)) { + platform_error_log("core_create_checkpoint_directory: failed to " + "allocate record extent %lu: %s\n", + slot, + platform_status_to_string(rc)); + return rc; + } + directory->record_addr[slot] = record_addr; + core_initialize_checkpoint_record_page(spl, record_addr); } - wait = 1; - cache_lock(spl->cc, super_page); - super = (core_super_block *)super_page->data; - uint64 old_root_addr = super->root_addr; + directory->checksum = core_checkpoint_directory_checksum(directory); + core_write_checkpoint_page( + spl, directory_addr, directory, sizeof(*directory)); - trunk_ondisk_node_handle root_handle; - trunk_init_root_handle(&spl->trunk_context, &root_handle); - uint64 root_addr = trunk_ondisk_node_handle_addr(&root_handle); - if (root_addr != 0) { - trunk_inc_ref(spl->al, root_addr); + /* + * Submit the newly initialized record and directory pages before the + * durable barrier. allocator_alloc() changes its refcount map only in + * memory; crash recovery deliberately rebuilds that map instead of relying + * on this publication. allocator_alloc_super_addr() does write the raw + * bootstrap mapping through the shared backing I/O handle, which the + * following durable barrier fdatasyncs with the directory page. + */ + rc = cache_writeback_fence(spl->cc); + if (!SUCCESS(rc)) { + return rc; } - super->root_addr = root_addr; - trunk_ondisk_node_handle_deinit(&root_handle); + return cache_durable_barrier(spl->cc); +} - if (spl->cfg.use_log) { - if (spl->log) { - super->log_addr = log_addr(spl->log); - super->log_meta_addr = log_meta_addr(spl->log); - } else { - super->log_addr = 0; - super->log_meta_addr = 0; +static platform_status +core_get_checkpoint_directory(core_handle *spl, + core_checkpoint_directory *directory) +{ + uint64 directory_addr; + platform_status rc = + allocator_get_super_addr(spl->al, spl->id, &directory_addr); + if (!SUCCESS(rc)) { + return rc; + } + + page_handle *page = + cache_get(spl->cc, directory_addr, TRUE, PAGE_TYPE_SUPERBLOCK); + memcpy(directory, page->data, sizeof(*directory)); + cache_unget(spl->cc, page); + + if (!core_checkpoint_directory_is_valid(spl, directory)) { + platform_error_log("core_get_checkpoint_directory: no compatible " + "checkpoint directory for root id %lu\n", + spl->id); + return STATUS_BAD_PARAM; + } + return STATUS_OK; +} + +static platform_status +core_load_checkpoint_records(core_handle *spl, + const core_checkpoint_directory *directory, + core_checkpoint_records *records) +{ + ZERO_CONTENTS(records); + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + page_handle *page = cache_get(spl->cc, + directory->record_addr[slot], + TRUE, + PAGE_TYPE_SUPERBLOCK); + memcpy(&records->record[slot], page->data, sizeof(records->record[slot])); + cache_unget(spl->cc, page); + + records->valid[slot] = core_checkpoint_record_is_valid( + spl, &records->record[slot], slot); + if (!records->valid[slot]) { + continue; + } + + if (!records->have_newest + || records->record[records->newest_slot].sequence + < records->record[slot].sequence) + { + records->have_newest = TRUE; + records->newest_slot = slot; + } + if (records->record[slot].unmounted + && (!records->have_newest_unmounted + || records->record[records->newest_unmounted_slot].sequence + < records->record[slot].sequence)) + { + records->have_newest_unmounted = TRUE; + records->newest_unmounted_slot = slot; } } - super->timestamp = platform_get_real_time(); - super->checkpointed = is_checkpoint; - super->unmounted = is_unmount; - super->checksum = - platform_checksum128(super, - sizeof(core_super_block) - sizeof(checksum128), - CORE_SUPER_CSUM_SEED); - - cache_mark_dirty(spl->cc, super_page); - cache_unlock(spl->cc, super_page); - cache_unclaim(spl->cc, super_page); - cache_unget(spl->cc, super_page); - cache_page_sync(spl->cc, super_page, TRUE, PAGE_TYPE_SUPERBLOCK); - - if (old_root_addr != 0 && !is_create) { + + if (records->valid[0] && records->valid[1] + && records->record[0].sequence == records->record[1].sequence) + { + platform_error_log("core_load_checkpoint_records: duplicate record " + "sequence %lu for root id %lu\n", + records->record[0].sequence, + spl->id); + return STATUS_BAD_PARAM; + } + return STATUS_OK; +} + +static void +core_destroy_checkpoint_record_extent(core_handle *spl, uint64 record_addr) +{ + refcount ref = + allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); + if (ref != AL_NO_REFS) { + platform_error_log("core_destroy_checkpoint_record_extent: record extent " + "%lu has unexpected refcount %u\n", + record_addr, + ref); + return; + } + + cache_extent_discard(spl->cc, record_addr, PAGE_TYPE_SUPERBLOCK); + ref = allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); + platform_assert(ref == AL_FREE); +} + +static void +core_destroy_checkpoint_storage(core_handle *spl) +{ + core_checkpoint_directory directory; + platform_status rc = core_get_checkpoint_directory(spl, &directory); + if (!SUCCESS(rc)) { + platform_error_log("core_destroy_checkpoint_storage: unable to load " + "checkpoint directory for root id %lu: %s\n", + spl->id, + platform_status_to_string(rc)); + return; + } + + core_checkpoint_records records; + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + platform_error_log("core_destroy_checkpoint_storage: unable to load " + "checkpoint records for root id %lu: %s\n", + spl->id, + platform_status_to_string(rc)); + return; + } + + /* + * Both valid slots own independent root references. Keeping the older + * record live makes it a real fallback if the next record write is torn; + * its reference is released only when that slot is successfully + * overwritten. This is clean destruction, so release both record owners. + */ + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + if (!records.valid[slot] || records.record[slot].root_addr == 0) { + continue; + } rc = trunk_dec_ref(spl->cfg.trunk_node_cfg, PROCESS_PRIVATE_HEAP_ID, spl->cc, spl->al, spl->ts, - old_root_addr); + records.record[slot].root_addr); if (!SUCCESS(rc)) { - platform_error_log("core_set_super_block: trunk_dec_ref failed for " - "old root addr %lu: %s\n", - old_root_addr, + platform_error_log("core_destroy_checkpoint_storage: failed to " + "release record %lu root %lu: %s\n", + slot, + records.record[slot].root_addr, platform_status_to_string(rc)); - return rc; } } + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + core_destroy_checkpoint_record_extent(spl, directory.record_addr[slot]); + } +} + +/* + *----------------------------------------------------------------------------- + * Checkpoint record functions + *----------------------------------------------------------------------------- + */ +static platform_status +core_capture_checkpoint_cut(core_handle *spl, + trunk_snapshot *snapshot, + bool32 *has_incorporated_generation, + uint64 *incorporated_generation) +{ + /* + * Incorporation publishes its generation and root while holding lookup + * exclusion before taking the trunk root lock. Take the checkpoint cut + * in the same order, so a record never combines a pre-incorporation + * generation with a post-incorporation root (or the converse). + */ + memtable_block_lookups(&spl->mt_ctxt); + uint64 retired_generation = memtable_generation_retired(&spl->mt_ctxt); + platform_status rc = trunk_snapshot_acquire(&spl->trunk_context, snapshot); + memtable_unblock_lookups(&spl->mt_ctxt); + if (!SUCCESS(rc)) { + return rc; + } + + *has_incorporated_generation = retired_generation != UINT64_MAX; + *incorporated_generation = *has_incorporated_generation + ? retired_generation + : 0; return STATUS_OK; } -static core_super_block * -core_get_super_block_if_valid(core_handle *spl, page_handle **super_page) +static platform_status +core_publish_checkpoint_record(core_handle *spl, + bool32 is_checkpoint, + bool32 is_unmount, + bool32 is_create) { - uint64 super_addr; - core_super_block *super; + uint64 old_root_addr; + platform_status rc; + trunk_snapshot snapshot; + bool32 has_incorporated_generation; + uint64 incorporated_generation; + core_checkpoint_directory directory; + core_checkpoint_records records; + uint64 target_slot; + core_checkpoint_record record; - platform_status rc = allocator_get_super_addr(spl->al, spl->id, &super_addr); - platform_assert_status_ok(rc); - *super_page = cache_get(spl->cc, super_addr, TRUE, PAGE_TYPE_SUPERBLOCK); - super = (core_super_block *)(*super_page)->data; - - if (!platform_checksum_is_equal( - super->checksum, - platform_checksum128(super, - sizeof(core_super_block) - sizeof(checksum128), - CORE_SUPER_CSUM_SEED))) + /* + * The snapshot, target-slot selection, durable record write, and old-slot + * release are one publication transaction. In particular, two concurrent + * publishers must never choose the same target slot or release the same + * former record owner. + */ + rc = platform_mutex_lock(&spl->checkpoint_lock); + if (!SUCCESS(rc)) { + return rc; + } + + rc = core_capture_checkpoint_cut(spl, + &snapshot, + &has_incorporated_generation, + &incorporated_generation); + if (!SUCCESS(rc)) { + goto unlock_checkpoint; + } + + /* + * The snapshot reference makes the root stable, but not necessarily + * durable. Drain only the cache intervals that existed at this cut before + * making a record that can name the root durable. This can incidentally + * persist newer log/data pages, but it does not seal or publish a logical + * durable-log tail; tail sync is a separate operation. + */ + rc = trunk_make_durable(&spl->trunk_context); + if (!SUCCESS(rc)) { + goto release_snapshot; + } + + if (is_create) { + rc = core_create_checkpoint_directory(spl, &directory); + } else { + rc = core_get_checkpoint_directory(spl, &directory); + } + if (!SUCCESS(rc)) { + platform_error_log("core_publish_checkpoint_record: failed to %s " + "checkpoint directory for root id %lu: %s\n", + is_create ? "create" : "load", + spl->id, + platform_status_to_string(rc)); + goto release_snapshot; + } + + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + goto release_snapshot; + } + + if (records.have_newest + && records.record[records.newest_slot].sequence == UINT64_MAX) { - cache_unget(spl->cc, *super_page); - *super_page = NULL; - return NULL; + rc = STATUS_LIMIT_EXCEEDED; + goto release_snapshot; + } + target_slot = records.have_newest ? records.newest_slot ^ 1 : 0; + /* + * Each valid record owns its root reference. This publication overwrites + * target_slot, so retain that slot's old root until the replacement page + * is durable, then release only the overwritten owner. The newest record + * remains independently live as the torn-write fallback. + */ + old_root_addr = records.valid[target_slot] + ? records.record[target_slot].root_addr + : 0; + + ZERO_CONTENTS(&record); + record.incorporated_generation = incorporated_generation; + record.has_incorporated_generation = has_incorporated_generation; + record.root_addr = snapshot.root_addr; + record.timestamp = platform_get_real_time(); + record.sequence = records.have_newest + ? records.record[records.newest_slot].sequence + 1 + : 1; + record.table_id = spl->id; + record.record_slot = target_slot; + record.checkpointed = is_checkpoint; + record.unmounted = is_unmount; + record.magic = CORE_CHECKPOINT_RECORD_MAGIC; + record.format_version = CORE_CHECKPOINT_FORMAT_VERSION; + + record.checksum = core_checkpoint_record_checksum(&record); + + core_write_checkpoint_page(spl, + directory.record_addr[target_slot], + &record, + sizeof(record)); + /* The record now owns this reference, even if the barrier reports failure. */ + snapshot.root_addr = 0; + + rc = cache_durable_barrier(spl->cc); + if (!SUCCESS(rc)) { + /* The new record may be durable, so retain its transferred root ref. */ + goto unlock_checkpoint; } - return super; -} + if (old_root_addr != 0) { + rc = trunk_dec_ref(spl->cfg.trunk_node_cfg, + PROCESS_PRIVATE_HEAP_ID, + spl->cc, + spl->al, + spl->ts, + old_root_addr); + if (!SUCCESS(rc)) { + platform_error_log("core_publish_checkpoint_record: trunk_dec_ref " + "failed for old root addr %lu: %s\n", + old_root_addr, + platform_status_to_string(rc)); + goto unlock_checkpoint; + } + } -static void -core_release_super_block(core_handle *spl, page_handle *super_page) -{ - cache_unget(spl->cc, super_page); + rc = STATUS_OK; + goto unlock_checkpoint; + +release_snapshot: + { + platform_status release_rc = + trunk_snapshot_release(&spl->trunk_context, &snapshot); + if (SUCCESS(rc) && !SUCCESS(release_rc)) { + rc = release_rc; + } + } + +unlock_checkpoint: + { + platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); + if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { + rc = unlock_rc; + } + } + return rc; } /* @@ -418,6 +848,7 @@ core_begin_memtable_insert(core_handle *spl, uint64 *generation, memtable **mt) static platform_status core_log_insert(core_handle *spl, + uint64 memtable_generation, key tuple_key, message msg, const btree_insert_results *insert_results) @@ -436,8 +867,11 @@ core_log_insert(core_handle *spl, merge_accumulator_is_null(&insert_results->msg_blob) ? msg : merge_accumulator_to_message(&insert_results->msg_blob); - int log_rc = - log_write(spl->log, tuple_key, log_msg, insert_results->leaf_generation); + int log_rc = log_write(spl->log, + tuple_key, + log_msg, + memtable_generation, + insert_results->leaf_generation); return log_rc == 0 ? STATUS_OK : (platform_status){.r = log_rc}; } @@ -1554,7 +1988,7 @@ core_insert(core_handle *spl, goto end_insert; } - rc = core_log_insert(spl, tuple_key, data, &insert_results); + rc = core_log_insert(spl, generation, tuple_key, data, &insert_results); if (!SUCCESS(rc)) { goto end_insert; } @@ -1879,18 +2313,25 @@ core_mkfs(core_handle *spl, spl->heap_id = hid; spl->ts = ts; + platform_status rc = core_checkpoint_lock_init(spl); + if (!SUCCESS(rc)) { + platform_error_log("core_mkfs: checkpoint lock initialization failed: %s\n", + platform_status_to_string(rc)); + return rc; + } + // set up the memtable context memtable_config *mt_cfg = &spl->cfg.mt_cfg; - platform_status rc = memtable_context_init(&spl->mt_ctxt, - spl->heap_id, - cc, - mt_cfg, - core_memtable_flush_virtual, - spl); + rc = memtable_context_init(&spl->mt_ctxt, + spl->heap_id, + cc, + mt_cfg, + core_memtable_flush_virtual, + spl); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: memtable_context_init failed: %s\n", platform_status_to_string(rc)); - return rc; + goto deinit_checkpoint_lock; } // set up the log @@ -1918,9 +2359,9 @@ core_mkfs(core_handle *spl, goto deinit_trunk_context; } - rc = core_set_super_block(spl, FALSE, FALSE, TRUE); + rc = core_publish_checkpoint_record(spl, FALSE, FALSE, TRUE); if (!SUCCESS(rc)) { - platform_error_log("core_mkfs: core_set_super_block failed: %s\n", + platform_error_log("core_mkfs: core_publish_checkpoint_record failed: %s\n", platform_status_to_string(rc)); goto deinit_stats; } @@ -1937,6 +2378,8 @@ core_mkfs(core_handle *spl, } deinit_memtable_context: memtable_context_deinit(&spl->mt_ctxt); +deinit_checkpoint_lock: + core_checkpoint_lock_deinit(spl); return rc; } @@ -1962,30 +2405,71 @@ core_mount(core_handle *spl, spl->heap_id = hid; spl->ts = ts; - // find the unmounted super block - uint64 root_addr = 0; - uint64 latest_timestamp = 0; - page_handle *super_page; - core_super_block *super = core_get_super_block_if_valid(spl, &super_page); - if (super != NULL) { - if (super->unmounted && super->timestamp > latest_timestamp) { - root_addr = super->root_addr; - latest_timestamp = super->timestamp; - } - core_release_super_block(spl, super_page); + platform_status rc = core_checkpoint_lock_init(spl); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: checkpoint lock initialization failed: %s\n", + platform_status_to_string(rc)); + return rc; } + /* + * Preserve the historical clean-only mount rule for this first format + * slice: an interrupted run is not replayed yet, so only an explicitly + * unmounted record supplies the root. We still validate both records and + * choose the newest clean one by sequence rather than wall-clock time. + */ + uint64 root_addr = 0; + bool32 has_incorporated_generation = FALSE; + uint64 incorporated_generation = 0; + core_checkpoint_directory directory; + core_checkpoint_records records; + rc = core_get_checkpoint_directory(spl, &directory); + if (!SUCCESS(rc)) { + goto deinit_checkpoint_lock; + } + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + goto deinit_checkpoint_lock; + } + if (!records.have_newest) { + platform_error_log("core_mount: checkpoint directory for root id %lu " + "has no valid records\n", + spl->id); + rc = STATUS_BAD_PARAM; + goto deinit_checkpoint_lock; + } + const core_checkpoint_record *record = + &records.record[records.newest_slot]; + if (!record->unmounted) { + /* + * This is an interrupted run. An older clean record is only an A/B + * torn-write fallback, not permission to silently discard the newer + * checkpoint and its log suffix. Do not overwrite its metadata before + * log replay and allocator reconstruction are wired. + */ + platform_error_log("core_mount: root id %lu requires crash recovery\n", + spl->id); + rc = STATUS_INVALID_STATE; + goto deinit_checkpoint_lock; + } + root_addr = record->root_addr; + has_incorporated_generation = record->has_incorporated_generation; + incorporated_generation = record->incorporated_generation; + memtable_config *mt_cfg = &spl->cfg.mt_cfg; - platform_status rc = memtable_context_init(&spl->mt_ctxt, - spl->heap_id, - cc, - mt_cfg, - core_memtable_flush_virtual, - spl); + rc = memtable_context_init_at_generation( + &spl->mt_ctxt, + spl->heap_id, + cc, + mt_cfg, + core_memtable_flush_virtual, + spl, + has_incorporated_generation ? incorporated_generation + 1 : 0); if (!SUCCESS(rc)) { - platform_error_log("core_mount: memtable_context_init failed: %s\n", + platform_error_log("core_mount: memtable_context_init_at_generation " + "failed: %s\n", platform_status_to_string(rc)); - return rc; + goto deinit_checkpoint_lock; } if (spl->cfg.use_log) { @@ -2012,9 +2496,9 @@ core_mount(core_handle *spl, goto deinit_trunk_context; } - rc = core_set_super_block(spl, FALSE, FALSE, FALSE); + rc = core_publish_checkpoint_record(spl, FALSE, FALSE, FALSE); if (!SUCCESS(rc)) { - platform_error_log("core_mount: core_set_super_block failed: %s\n", + platform_error_log("core_mount: core_publish_checkpoint_record failed: %s\n", platform_status_to_string(rc)); goto deinit_stats; } @@ -2031,6 +2515,8 @@ core_mount(core_handle *spl, } deinit_memtable_context: memtable_context_deinit(&spl->mt_ctxt); +deinit_checkpoint_lock: + core_checkpoint_lock_deinit(spl); return rc; } @@ -2082,9 +2568,11 @@ core_report_unincorporated_memtables(core_handle *spl) /* * This function is only safe to call when all other calls to spl have returned. + * It intentionally leaves the memtable and log contexts live: the clean + * checkpoint record needs both after final incorporation has quiesced. */ -void -core_prepare_for_shutdown(core_handle *spl) +static void +core_quiesce_for_shutdown(core_handle *spl) { // write current memtable to disk // (any others must already be flushing/flushed) @@ -2104,13 +2592,19 @@ core_prepare_for_shutdown(core_handle *spl) platform_assert_status_ok(rc); core_report_unincorporated_memtables(spl); +} - // destroy memtable context (and its memtables) +static void +core_teardown_after_shutdown(core_handle *spl) +{ + // Keep this after checkpoint publication: it supplies the generation cut. memtable_context_deinit(&spl->mt_ctxt); - // release the log + // Keep the log alive through clean-record publication. A later explicit + // tail-sync protocol will own its immutable log metadata separately. if (spl->cfg.use_log) { platform_free(spl->heap_id, spl->log); + spl->log = NULL; } // flush all dirty pages in the cache @@ -2126,14 +2620,21 @@ core_unmount(core_handle *spl) { platform_status rc; - core_prepare_for_shutdown(spl); - rc = core_set_super_block(spl, FALSE, TRUE, FALSE); + /* + * Quiescing leaves the memtable and log contexts live so publication can + * atomically capture the retired generation and root, then record log + * metadata. Teardown is safe regardless of publication success. + */ + core_quiesce_for_shutdown(spl); + rc = core_publish_checkpoint_record(spl, FALSE, TRUE, FALSE); if (!SUCCESS(rc)) { - platform_error_log("core_unmount: failed to update super block: %s\n", + platform_error_log("core_unmount: failed to publish checkpoint record: %s\n", platform_status_to_string(rc)); } + core_teardown_after_shutdown(spl); trunk_context_deinit(&spl->trunk_context); core_destroy_stats(spl); + core_checkpoint_lock_deinit(spl); return rc; } @@ -2143,12 +2644,16 @@ core_unmount(core_handle *spl) void core_destroy(core_handle *spl) { - core_prepare_for_shutdown(spl); + core_quiesce_for_shutdown(spl); + core_teardown_after_shutdown(spl); + /* Records own trunk references and their two dedicated record extents. */ + core_destroy_checkpoint_storage(spl); trunk_context_deinit(&spl->trunk_context); // clear out this splinter table from the meta page. allocator_remove_super_addr(spl->al, spl->id); core_destroy_stats(spl); + core_checkpoint_lock_deinit(spl); } @@ -2181,27 +2686,58 @@ core_print_space_use(platform_log_handle *log_handle, core_handle *spl) /* * core_print_super_block() * - * Fetch a super-block for a running Splinter instance, and print its - * contents. + * Print the fixed checkpoint directory and both independently written record + * slots for a running Splinter instance. */ void core_print_super_block(platform_log_handle *log_handle, core_handle *spl) { - page_handle *super_page; - core_super_block *super = core_get_super_block_if_valid(spl, &super_page); - if (super == NULL) { + core_checkpoint_directory directory; + platform_status rc = core_get_checkpoint_directory(spl, &directory); + if (!SUCCESS(rc)) { + platform_log(log_handle, + "No compatible checkpoint directory for root id %lu\n", + spl->id); + return; + } + + core_checkpoint_records records; + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + platform_log(log_handle, + "Unable to load checkpoint records for root id %lu: %s\n", + spl->id, + platform_status_to_string(rc)); return; } - platform_log(log_handle, "Superblock root_addr=%lu {\n", super->root_addr); - platform_log(log_handle, "log_meta_addr=%lu\n", super->log_meta_addr); platform_log(log_handle, - "timestamp=%lu, checkpointed=%d, unmounted=%d\n", - super->timestamp, - super->checkpointed, - super->unmounted); + "Checkpoint directory root_id=%lu record_addr=[%lu, %lu] {\n", + directory.table_id, + directory.record_addr[0], + directory.record_addr[1]); + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + if (!records.valid[slot]) { + platform_log(log_handle, " record[%lu]: invalid\n", slot); + continue; + } + core_checkpoint_record *record = &records.record[slot]; + platform_log(log_handle, + " record[%lu]: sequence=%lu root_addr=%lu " + "has_incorporated_generation=%d " + "incorporated_generation=%lu " + "timestamp=%lu " + "checkpointed=%d unmounted=%d\n", + slot, + record->sequence, + record->root_addr, + record->has_incorporated_generation, + record->incorporated_generation, + record->timestamp, + record->checkpointed, + record->unmounted); + } platform_log(log_handle, "}\n\n"); - core_release_super_block(spl, super_page); } // clang-format off diff --git a/src/core.h b/src/core.h index 94c10f97d..997c06222 100644 --- a/src/core.h +++ b/src/core.h @@ -119,6 +119,10 @@ struct core_handle { trunk_context trunk_context; memtable_context mt_ctxt; + /* Serializes snapshot cuts and A/B checkpoint-record publication. */ + platform_mutex checkpoint_lock; + bool32 checkpoint_lock_initialized; + core_stats *stats; core_compacted_memtable compacted_memtable[MAX_MEMTABLES]; diff --git a/src/log.h b/src/log.h index 325d07648..3dd27172d 100644 --- a/src/log.h +++ b/src/log.h @@ -16,16 +16,55 @@ typedef struct log_handle log_handle; typedef struct log_iterator log_iterator; typedef struct log_config log_config; +/* + * Identity of one mini-allocator-backed log stream. It is sufficient for a + * higher-level checkpoint record to describe a stream, but not by itself a + * durable descriptor: core will later persist this information in an + * independently checksummed log-segment record. + */ +typedef struct log_segment_info { + uint64 addr; + uint64 meta_addr; + uint64 magic; +} log_segment_info; + typedef int (*log_write_fn)(log_handle *log, key tuple_key, message data, - uint64 generation); + uint64 memtable_generation, + uint64 leaf_generation); +/* + * Finalize the current append pages into checksummed, immutable pages. + * + * The caller must exclude concurrent log_write() calls until it has taken the + * cache writeback fence that is to make the pages durable. It must also + * serialize concurrent log_seal() calls. seal() itself does not issue I/O or + * a durable barrier. + * + * This is deliberately not a persisted replay-boundary descriptor. A log + * implementation whose metadata can grow after sealing needs an additional + * boundary in the checkpoint record to exclude those later entries. + */ +typedef platform_status (*log_seal_fn)(log_handle *log); +/* + * Detach the current stream after sealing it and prepare a distinct fresh + * stream. The caller must exclude writes throughout the operation and make + * the returned identities durable before allowing writes to the fresh stream. + * A zero sealed.meta_addr means the old stream was empty and discarded. + * Rotation alone does not advance the logical durable-log tail: that requires + * a separate, durable tail/manifest publication by the caller. + */ +typedef platform_status (*log_rotate_fn)(log_handle *log, + log_segment_info *sealed, + log_segment_info *fresh); typedef void (*log_release_fn)(log_handle *log); typedef uint64 (*log_addr_fn)(log_handle *log); typedef uint64 (*log_magic_fn)(log_handle *log); typedef struct log_ops { log_write_fn write; + log_seal_fn seal; + log_rotate_fn rotate; log_release_fn release; log_addr_fn addr; log_addr_fn meta_addr; @@ -38,9 +77,32 @@ struct log_handle { }; static inline int -log_write(log_handle *log, key tuple_key, message data, uint64 generation) +log_write(log_handle *log, + key tuple_key, + message data, + uint64 memtable_generation, + uint64 leaf_generation) +{ + return log->ops->write( + log, tuple_key, data, memtable_generation, leaf_generation); +} + +/* + * Finalize the log's current append pages. See log_seal_fn for the required + * exclusion, boundary, and durability ordering. + */ +static inline platform_status +log_seal(log_handle *log) +{ + return log->ops->seal(log); +} + +static inline platform_status +log_rotate(log_handle *log, + log_segment_info *sealed, + log_segment_info *fresh) { - return log->ops->write(log, tuple_key, data, generation); + return log->ops->rotate(log, sealed, fresh); } static inline void diff --git a/src/memtable.c b/src/memtable.c index 41482e68d..e8c8d95e0 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -80,6 +80,20 @@ memtable_end_insert(memtable_context *ctxt) batch_rwlock_unget(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } +void +memtable_block_inserts(memtable_context *ctxt) +{ + batch_rwlock_get(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_claim_loop(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_lock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); +} + +void +memtable_unblock_inserts(memtable_context *ctxt) +{ + batch_rwlock_full_unlock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); +} + static inline bool32 memtable_try_begin_insert_rotation(memtable_context *ctxt) { @@ -97,20 +111,6 @@ memtable_end_insert_rotation(memtable_context *ctxt) batch_rwlock_unclaim(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } -static inline void -memtable_begin_raw_rotation(memtable_context *ctxt) -{ - batch_rwlock_get(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); - batch_rwlock_claim_loop(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); - batch_rwlock_lock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); -} - -static inline void -memtable_end_raw_rotation(memtable_context *ctxt) -{ - batch_rwlock_full_unlock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); -} - void memtable_begin_lookup(memtable_context *ctxt) { @@ -297,7 +297,7 @@ memtable_mark_incorporation_failed(memtable *mt, platform_status status) uint64 memtable_force_finalize(memtable_context *ctxt) { - memtable_begin_raw_rotation(ctxt); + memtable_block_inserts(ctxt); uint64 generation = ctxt->generation; uint64 mt_no = generation % ctxt->cfg.max_memtables; @@ -308,7 +308,7 @@ memtable_force_finalize(memtable_context *ctxt) <= ctxt->cfg.max_memtables); memtable_mark_empty(ctxt); - memtable_end_raw_rotation(ctxt); + memtable_unblock_inserts(ctxt); return current_generation; } @@ -357,12 +357,13 @@ memtable_deinit(cache *cc, memtable *mt) } platform_status -memtable_context_init(memtable_context *ctxt, - platform_heap_id hid, - cache *cc, - memtable_config *cfg, - process_fn process, - void *process_ctxt) +memtable_context_init_at_generation(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn process, + void *process_ctxt, + uint64 first_generation) { platform_status rc; ZERO_CONTENTS(ctxt); @@ -370,14 +371,22 @@ memtable_context_init(memtable_context *ctxt, ctxt->cfg = *cfg; ctxt->hid = hid; - if (MAX_MEMTABLES < cfg->max_memtables) { - platform_error_log("Configured number of memtables (%lu) exceeds max " - "supported memtables (%d)\n", + if (cfg->max_memtables == 0 || MAX_MEMTABLES < cfg->max_memtables) { + platform_error_log("Configured number of memtables (%lu) must be " + "between 1 and %d\n", cfg->max_memtables, MAX_MEMTABLES); return STATUS_BAD_PARAM; } + if (first_generation > (uint64)-1 - cfg->max_memtables) { + platform_error_log("First memtable generation (%lu) leaves too little " + "space for %lu memtable slots\n", + first_generation, + cfg->max_memtables); + return STATUS_BAD_PARAM; + } + rc = platform_mutex_init( &ctxt->incorporation_mutex, platform_get_module_id(), hid); if (!SUCCESS(rc)) { @@ -394,14 +403,25 @@ memtable_context_init(memtable_context *ctxt, batch_rwlock_init(&ctxt->rwlock); - for (uint64 mt_no = 0; mt_no < cfg->max_memtables; mt_no++) { - uint64 generation = mt_no; + for (uint64 generation_offset = 0; + generation_offset < cfg->max_memtables; + generation_offset++) + { + uint64 generation = first_generation + generation_offset; + uint64 mt_no = generation % cfg->max_memtables; memtable_init(&ctxt->mt[mt_no], cc, cfg, generation); } - ctxt->generation = 0; - ctxt->generation_to_incorporate = 0; - ctxt->generation_retired = (uint64)-1; + ctxt->generation = first_generation; + ctxt->generation_to_incorporate = first_generation; + /* + * A fresh database has no incorporated generation. The UINT64_MAX + * sentinel preserves the existing unsigned generation-ring arithmetic. + * Otherwise, the checkpoint has incorporated every generation before + * first_generation. + */ + ctxt->generation_retired = first_generation == 0 ? (uint64)-1 + : first_generation - 1; ctxt->is_empty = TRUE; @@ -411,6 +431,18 @@ memtable_context_init(memtable_context *ctxt, return STATUS_OK; } +platform_status +memtable_context_init(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn process, + void *process_ctxt) +{ + return memtable_context_init_at_generation( + ctxt, hid, cc, cfg, process, process_ctxt, 0); +} + void memtable_context_deinit(memtable_context *ctxt) { diff --git a/src/memtable.h b/src/memtable.h index 7c82aadd7..bfea7002d 100644 --- a/src/memtable.h +++ b/src/memtable.h @@ -159,6 +159,17 @@ memtable_maybe_rotate_and_begin_insert(memtable_context *ctxt, void memtable_end_insert(memtable_context *ctxt); +/* + * Exclude all inserts, including an insert that has already acquired its + * shared insert lock. A checkpoint uses this around sealing the log prefix + * that describes its snapshot. Must be paired with memtable_unblock_inserts. + */ +void +memtable_block_inserts(memtable_context *ctxt); + +void +memtable_unblock_inserts(memtable_context *ctxt); + void memtable_begin_lookup(memtable_context *ctxt); @@ -211,6 +222,24 @@ memtable_context_init(memtable_context *ctxt, process_fn process, void *process_ctxt); +/* + * Initialize the reusable memtable ring at first_generation. Recovery passes + * the checkpoint's incorporated generation plus one; the trunk is understood + * to have incorporated all earlier generations. In particular, slot + * first_generation % max_memtables is the active memtable and the other slots + * represent the following logical generations. first_generation == 0 is the + * fresh-database case and has no incorporated generation. A caller must not + * wrap a checkpoint generation of UINT64_MAX into zero. + */ +platform_status +memtable_context_init_at_generation(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn process, + void *process_ctxt, + uint64 first_generation); + void memtable_context_deinit(memtable_context *ctxt); diff --git a/src/mini_allocator.c b/src/mini_allocator.c index 1099641f1..fa350f786 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -917,6 +917,247 @@ mini_prefetch(cache *cc, page_type type, uint64 meta_head) mini_for_each(cc, meta_head, type, mini_prefetch_extent, NULL); } +/* + * ----------------------------------------------------------------------------- + * mini_recovery_walk -- Read-only mini-allocator extent enumeration. + * ----------------------------------------------------------------------------- + */ + +static platform_status +mini_recovery_corruption(const char *reason, uint64 addr) +{ + platform_error_log("Malformed mini allocator metadata: %s (addr=%lu).\n", + reason, + addr); + return STATUS_INVALID_STATE; +} + +static bool32 +mini_recovery_valid_geometry(const allocator_config *cfg) +{ + if (cfg == NULL || cfg->io_cfg == NULL || cfg->capacity == 0 + || cfg->io_cfg->page_size == 0 || cfg->io_cfg->extent_size == 0 + || cfg->io_cfg->page_size + < offsetof(mini_meta_hdr, entry_buffer) + || cfg->io_cfg->extent_size < cfg->io_cfg->page_size + || cfg->io_cfg->extent_size % cfg->io_cfg->page_size != 0 + || cfg->capacity % cfg->io_cfg->extent_size != 0) + { + return FALSE; + } + + return TRUE; +} + +static bool32 +mini_recovery_valid_page_addr(const allocator_config *cfg, uint64 addr) +{ + uint64 page_size = cfg->io_cfg->page_size; + + return addr != 0 && addr % page_size == 0 + && addr <= cfg->capacity - page_size; +} + +static bool32 +mini_recovery_valid_extent_addr(const allocator_config *cfg, uint64 addr) +{ + uint64 extent_size = cfg->io_cfg->extent_size; + + return addr != 0 && addr % extent_size == 0 + && addr <= cfg->capacity - extent_size; +} + +static uint64 +mini_recovery_extent_base_addr(const allocator_config *cfg, uint64 addr) +{ + return addr - addr % cfg->io_cfg->extent_size; +} + +static platform_status +mini_recovery_validate_meta_header(const mini_meta_hdr *hdr, + uint64 page_size, + uint64 expected_prev, + uint64 meta_addr) +{ + uint64 first_entry_offset = offsetof(mini_meta_hdr, entry_buffer); + uint64 max_entries = (page_size - first_entry_offset) / sizeof(meta_entry); + if (hdr->prev_meta_addr != expected_prev) { + return mini_recovery_corruption("metadata prev link is not reciprocal", + meta_addr); + } + + if (hdr->num_entries > max_entries) { + return mini_recovery_corruption("metadata entry count exceeds page", + meta_addr); + } + + uint64 expected_pos = first_entry_offset + + (uint64)hdr->num_entries * sizeof(meta_entry); + if (hdr->pos != expected_pos) { + return mini_recovery_corruption("metadata entry position is invalid", + meta_addr); + } + + return STATUS_OK; +} + +static platform_status +mini_recovery_validate_next_meta_addr(const allocator_config *cfg, + uint64 meta_addr, + uint64 next_meta_addr) +{ + if (next_meta_addr == 0) { + return STATUS_OK; + } + + if (!mini_recovery_valid_page_addr(cfg, next_meta_addr)) { + return mini_recovery_corruption("metadata next link is not a page", + next_meta_addr); + } + + uint64 extent_size = cfg->io_cfg->extent_size; + uint64 page_size = cfg->io_cfg->page_size; + uint64 offset = meta_addr % extent_size; + if (offset == extent_size - page_size) { + if (next_meta_addr % extent_size != 0) { + return mini_recovery_corruption( + "metadata extent transition does not start at an extent base", + next_meta_addr); + } + } else if (next_meta_addr != meta_addr + page_size) { + return mini_recovery_corruption( + "metadata pages do not advance contiguously within an extent", + next_meta_addr); + } + + return STATUS_OK; +} + +platform_status +mini_recovery_walk(cache *cc, + uint64 meta_head, + page_type meta_type, + mini_recovery_visit_fn visit, + void *arg) +{ + if (cc == NULL || visit == NULL + || meta_type < PAGE_TYPE_FIRST || meta_type >= NUM_PAGE_TYPES) + { + return STATUS_BAD_PARAM; + } + + allocator *al = cache_get_allocator(cc); + if (al == NULL) { + return STATUS_BAD_PARAM; + } + allocator_config *cfg = allocator_get_config(al); + if (!mini_recovery_valid_geometry(cfg)) { + return STATUS_BAD_PARAM; + } + if (!mini_recovery_valid_page_addr(cfg, meta_head)) { + return mini_recovery_corruption("metadata head is not a page", meta_head); + } + + uint64 meta_addr = meta_head; + uint64 expected_prev = 0; + uint64 prior_meta_extent = (uint64)-1; + uint64 max_meta_pages = cfg->capacity / cfg->io_cfg->page_size; + + for (uint64 page_count = 0; meta_addr != 0; page_count++) { + if (page_count == max_meta_pages) { + return mini_recovery_corruption("metadata chain exceeds disk pages", + meta_addr); + } + + /* + * Report a metadata extent before reading it. The normal mini metadata + * layout is contiguous within an extent, so this is once per physical + * metadata extent; next-link and reciprocal-prev validation below reject + * a malformed chain before it can complete a loop. + */ + uint64 meta_extent = mini_recovery_extent_base_addr(cfg, meta_addr); + if (!mini_recovery_valid_extent_addr(cfg, meta_extent)) { + return mini_recovery_corruption("metadata extent is out of range", + meta_extent); + } + if (meta_extent != prior_meta_extent) { + platform_status rc = visit(meta_extent, + meta_type, + MINI_RECOVERY_EXTENT_METADATA, + MINI_RECOVERY_METADATA_BATCH, + arg); + if (!SUCCESS(rc)) { + return rc; + } + prior_meta_extent = meta_extent; + } + + page_handle *meta_page = cache_get(cc, meta_addr, TRUE, meta_type); + if (meta_page == NULL) { + return STATUS_IO_ERROR; + } + + mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; + platform_status rc = mini_recovery_validate_meta_header( + hdr, cfg->io_cfg->page_size, expected_prev, meta_addr); + if (!SUCCESS(rc)) { + cache_unget(cc, meta_page); + return rc; + } + + uint64 next_meta_addr = hdr->next_meta_addr; + rc = mini_recovery_validate_next_meta_addr( + cfg, meta_addr, next_meta_addr); + if (!SUCCESS(rc)) { + cache_unget(cc, meta_page); + return rc; + } + + meta_entry *entry = first_entry(meta_page); + for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { + uint64 batch = meta_entry_batch(entry); + page_type extent_type = meta_entry_type(entry); + uint64 extent_number = + entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); + + if (batch >= MINI_MAX_BATCHES + || extent_type < PAGE_TYPE_FIRST || extent_type >= NUM_PAGE_TYPES + || extent_number == 0 + || extent_number > UINT64_MAX / cfg->io_cfg->extent_size) + { + cache_unget(cc, meta_page); + return mini_recovery_corruption("metadata extent entry is invalid", + meta_addr); + } + + uint64 extent_addr = extent_number * cfg->io_cfg->extent_size; + if (!mini_recovery_valid_extent_addr(cfg, extent_addr)) { + cache_unget(cc, meta_page); + return mini_recovery_corruption("metadata extent is out of range", + extent_addr); + } + + rc = visit(extent_addr, + extent_type, + MINI_RECOVERY_EXTENT_DATA, + batch, + arg); + if (!SUCCESS(rc)) { + cache_unget(cc, meta_page); + return rc; + } + + entry = next_entry(entry); + } + + cache_unget(cc, meta_page); + expected_prev = meta_addr; + meta_addr = next_meta_addr; + } + + return STATUS_OK; +} + /* *----------------------------------------------------------------------------- * mini_meta_cursor -- cursor over a mini_allocator's extent entries. diff --git a/src/mini_allocator.h b/src/mini_allocator.h index 1a7995b7a..f596d9864 100644 --- a/src/mini_allocator.h +++ b/src/mini_allocator.h @@ -113,6 +113,57 @@ mini_unblock_dec_ref(cache *cc, uint64 meta_head); void mini_prefetch(cache *cc, page_type type, uint64 meta_head); +/* + * mini_recovery_walk -- + * + * Enumerate the physical extents reachable from a finalized mini + * allocator without looking at allocator refcounts. This is the + * discovery primitive used by crash recovery to rebuild those refcounts. + * + * The callback is invoked once for each metadata extent and once for + * every data-extent entry in the on-disk mini metadata stream. Data + * entries are deliberately not deduplicated: repeated entries represent + * repeated references in the mini allocator and are reported exactly as + * recorded. Metadata entries have batch + * MINI_RECOVERY_METADATA_BATCH. + * + * The walker validates the metadata-page chain, page-header bounds, page + * types, batches, and extent addresses before using them. A malformed + * on-disk stream returns STATUS_INVALID_STATE. Callback failures are + * returned unchanged. The walker performs no writes and does not use + * allocator refcounts to decide what to traverse. + * + * The callback is called for a metadata extent before the walker reads a + * page from that extent. That ordering lets an allocator-recovery caller + * establish temporary ownership before cache_get() performs its debug + * allocation checks. + * + * This is a physical enumeration only. Recovering logical reference + * multiplicity, and deduplicating references shared by distinct mini + * allocator roots, remains the responsibility of the higher-level + * trunk/log recovery walker. + */ +typedef enum mini_recovery_extent_kind { + MINI_RECOVERY_EXTENT_METADATA, + MINI_RECOVERY_EXTENT_DATA, +} mini_recovery_extent_kind; + +#define MINI_RECOVERY_METADATA_BATCH ((uint64)-1) + +typedef platform_status (*mini_recovery_visit_fn)( + uint64 extent_addr, + page_type type, + mini_recovery_extent_kind kind, + uint64 batch, + void *arg); + +platform_status +mini_recovery_walk(cache *cc, + uint64 meta_head, + page_type meta_type, + mini_recovery_visit_fn visit, + void *arg); + /* * mini_meta_cursor: a non-blocking cursor over the extent entries of a * finalized mini_allocator. Entries from all batches are interleaved in diff --git a/src/platform_linux/laio.c b/src/platform_linux/laio.c index 87e6c3f57..6988514b3 100644 --- a/src/platform_linux/laio.c +++ b/src/platform_linux/laio.c @@ -213,6 +213,35 @@ laio_write(io_handle *ioh, void *buf, uint64 bytes, uint64 addr) return STATUS_IO_ERROR; } +/* + *-------------------------------------------------------------------------- + * laio_durable_barrier -- + * + * Make completed writes durable. Waiting for asynchronous I/O only tells us + * that the kernel accepted the writes; fdatasync() supplies the persistence + * ordering needed by checkpoint publication on buffered files. + *-------------------------------------------------------------------------- + */ +static platform_status +laio_durable_barrier(io_handle *ioh) +{ + laio_handle *io = (laio_handle *)ioh; + int ret; + + do { + ret = fdatasync(io->fd); + } while (ret != 0 && errno == EINTR); + + if (ret == 0) { + return STATUS_OK; + } + + int saved_errno = errno; + platform_error_log("laio_durable_barrier: fdatasync failed: %s\n", + strerror(saved_errno)); + return CONST_STATUS(saved_errno); +} + /* * Accessor method: Return opaque handle to IO-context setup by io_setup(). */ @@ -674,6 +703,7 @@ static io_ops laio_ops = { .async_state_init = laio_async_state_init, .cleanup = laio_cleanup, .wait_all = laio_wait_all, + .durable_barrier = laio_durable_barrier, .print_stats = laio_print_stats, .reset_stats = laio_reset_stats, }; diff --git a/src/platform_linux/platform_io.h b/src/platform_linux/platform_io.h index 1d96bf1a1..3ecd12e09 100644 --- a/src/platform_linux/platform_io.h +++ b/src/platform_linux/platform_io.h @@ -78,6 +78,7 @@ typedef platform_status (*io_async_state_init_fn)(io_async_state *state, typedef void (*io_cleanup_fn)(io_handle *io, uint64 count); typedef void (*io_wait_all_fn)(io_handle *io); +typedef platform_status (*io_durable_barrier_fn)(io_handle *io); typedef void (*io_register_thread_fn)(io_handle *io); typedef void (*io_deregister_thread_fn)(io_handle *io); typedef bool32 (*io_max_latency_elapsed_fn)(io_handle *io, timestamp ts); @@ -95,6 +96,7 @@ typedef struct io_ops { io_async_state_init_fn async_state_init; io_cleanup_fn cleanup; io_wait_all_fn wait_all; + io_durable_barrier_fn durable_barrier; io_register_thread_fn register_thread; io_deregister_thread_fn deregister_thread; io_max_latency_elapsed_fn max_latency_elapsed; @@ -204,6 +206,21 @@ io_wait_all(io_handle *io) return io->ops->wait_all(io); } +/* + *-------------------------------------------------------------------------- + * io_durable_barrier + * + * Ensure that writes which completed before this call survive a power loss. + * This is deliberately distinct from io_wait_all(), which only waits for + * asynchronous I/O completion. + *-------------------------------------------------------------------------- + */ +static inline platform_status +io_durable_barrier(io_handle *io) +{ + return io->ops->durable_barrier(io); +} + static inline void io_register_thread(io_handle *io) { diff --git a/src/rc_allocator.c b/src/rc_allocator.c index 70257dc59..7a365fa2a 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -17,7 +17,15 @@ #include "platform_typed_alloc.h" #include "poison.h" -#define RC_ALLOCATOR_META_PAGE_CSUM_SEED (2718281828) +#define RC_ALLOCATOR_META_PAGE_CSUM_SEED (2718281828) +#define RC_ALLOCATOR_CLEAN_STATE_CSUM_SEED (2718281829) + +#define RC_ALLOCATOR_FORMAT_MAGIC (0x534442414C4C4F43ULL) // SDBALLOC +#define RC_ALLOCATOR_FORMAT_VERSION (1) + +#define RC_ALLOCATOR_CLEAN_STATE_MAGIC (0x534442434C45414EULL) // SDBCLEAN +#define RC_ALLOCATOR_CLEAN_STATE_VERSION (1) +#define RC_ALLOCATOR_CLEAN_STATE_SLOTS (2) /* * Base offset from where the allocator starts. Currently hard coded to 0. @@ -35,6 +43,28 @@ */ #define SHOULD_TRACE(addr) (0) // Do not trace anything +/* + * A/B clean-state records live in two fixed extents after the refcount map. + * They are deliberately separate from the sole allocator bootstrap page: a + * torn state update must leave an older valid state record available for the + * next mount or rebuild. + */ +typedef struct ONDISK rc_allocator_clean_state { + uint64 magic; + uint64 format_version; + uint64 sequence; + bool32 clean_shutdown; + checksum128 checksum; +} rc_allocator_clean_state; + +typedef struct rc_allocator_clean_states { + rc_allocator_clean_state state[RC_ALLOCATOR_CLEAN_STATE_SLOTS]; + bool32 valid[RC_ALLOCATOR_CLEAN_STATE_SLOTS]; + bool32 have_newest; + uint64 newest_slot; + bool32 duplicate_sequence; +} rc_allocator_clean_states; + /* *------------------------------------------------------------------------------ * Function declarations and virtual trampolines @@ -231,6 +261,16 @@ rc_allocator_meta_page_checksum(const rc_allocator_meta_page *meta_page) RC_ALLOCATOR_META_PAGE_CSUM_SEED); } +static platform_status +rc_allocator_write_meta_page(rc_allocator *al) +{ + al->meta_page->checksum = rc_allocator_meta_page_checksum(al->meta_page); + return io_write(al->io, + al->meta_page, + al->cfg->io_cfg->page_size, + RC_ALLOCATOR_BASE_OFFSET); +} + static disk_geometry rc_allocator_config_get_disk_geometry(allocator_config *cfg) { @@ -241,6 +281,260 @@ rc_allocator_config_get_disk_geometry(allocator_config *cfg) }; } +static uint64 +rc_allocator_refcount_buffer_size(const allocator_config *cfg) +{ + uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); + return ROUNDUP(buffer_size, cfg->io_cfg->page_size); +} + +static uint64 +rc_allocator_refcount_extent_count(const allocator_config *cfg) +{ + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); + return (buffer_size + cfg->io_cfg->extent_size - 1) + / cfg->io_cfg->extent_size; +} + +static uint64 +rc_allocator_clean_state_extent_no(const allocator_config *cfg, uint64 slot) +{ + platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); + return 1 + rc_allocator_refcount_extent_count(cfg) + slot; +} + +static uint64 +rc_allocator_clean_state_addr(const allocator_config *cfg, uint64 slot) +{ + return rc_allocator_clean_state_extent_no(cfg, slot) + * cfg->io_cfg->extent_size; +} + +static uint64 +rc_allocator_reserved_extent_count(const allocator_config *cfg) +{ + return 1 + rc_allocator_refcount_extent_count(cfg) + + RC_ALLOCATOR_CLEAN_STATE_SLOTS; +} + +static checksum128 +rc_allocator_clean_state_checksum(const rc_allocator_clean_state *state) +{ + return platform_checksum128(state, + offsetof(rc_allocator_clean_state, checksum), + RC_ALLOCATOR_CLEAN_STATE_CSUM_SEED); +} + +static bool32 +rc_allocator_clean_state_is_valid(const rc_allocator_clean_state *state) +{ + return state->magic == RC_ALLOCATOR_CLEAN_STATE_MAGIC + && state->format_version == RC_ALLOCATOR_CLEAN_STATE_VERSION + && state->sequence != 0 + && (state->clean_shutdown == FALSE || state->clean_shutdown == TRUE) + && platform_checksum_is_equal( + state->checksum, rc_allocator_clean_state_checksum(state)); +} + +static platform_status +rc_allocator_read_clean_state(rc_allocator *al, + uint64 slot, + rc_allocator_clean_state *state, + bool32 *valid) +{ + platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); + platform_assert(sizeof(*state) <= al->cfg->io_cfg->page_size); + + buffer_handle buffer; + platform_status rc = + platform_buffer_init(&buffer, al->cfg->io_cfg->page_size); + if (!SUCCESS(rc)) { + return rc; + } + + void *page = platform_buffer_getaddr(&buffer); + rc = io_read(al->io, + page, + al->cfg->io_cfg->page_size, + rc_allocator_clean_state_addr(al->cfg, slot)); + if (SUCCESS(rc)) { + memcpy(state, page, sizeof(*state)); + *valid = rc_allocator_clean_state_is_valid(state); + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +static platform_status +rc_allocator_write_clean_state(rc_allocator *al, + uint64 slot, + const rc_allocator_clean_state *state, + bool32 durable) +{ + platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); + platform_assert(sizeof(*state) <= al->cfg->io_cfg->page_size); + + buffer_handle buffer; + platform_status rc = + platform_buffer_init(&buffer, al->cfg->io_cfg->page_size); + if (!SUCCESS(rc)) { + return rc; + } + + void *page = platform_buffer_getaddr(&buffer); + memset(page, 0, al->cfg->io_cfg->page_size); + memcpy(page, state, sizeof(*state)); + rc = io_write(al->io, + page, + al->cfg->io_cfg->page_size, + rc_allocator_clean_state_addr(al->cfg, slot)); + if (SUCCESS(rc) && durable) { + rc = io_durable_barrier(al->io); + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +static platform_status +rc_allocator_load_clean_states(rc_allocator *al, + rc_allocator_clean_states *states) +{ + ZERO_CONTENTS(states); + for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { + platform_status rc = rc_allocator_read_clean_state( + al, slot, &states->state[slot], &states->valid[slot]); + if (!SUCCESS(rc)) { + return rc; + } + + if (!states->valid[slot]) { + continue; + } + if (!states->have_newest + || states->state[states->newest_slot].sequence + < states->state[slot].sequence) + { + states->have_newest = TRUE; + states->newest_slot = slot; + } else if (states->state[states->newest_slot].sequence + == states->state[slot].sequence) + { + states->duplicate_sequence = TRUE; + } + } + return STATUS_OK; +} + +static platform_status +rc_allocator_publish_clean_state(rc_allocator *al, bool32 clean_shutdown) +{ + rc_allocator_clean_states states; + platform_status rc = rc_allocator_load_clean_states(al, &states); + if (!SUCCESS(rc)) { + return rc; + } + + if (states.have_newest + && states.state[states.newest_slot].sequence == UINT64_MAX) + { + return STATUS_LIMIT_EXCEEDED; + } + + uint64 target_slot = states.have_newest ? states.newest_slot ^ 1 : 0; + rc_allocator_clean_state state; + ZERO_CONTENTS(&state); + state.magic = RC_ALLOCATOR_CLEAN_STATE_MAGIC; + state.format_version = RC_ALLOCATOR_CLEAN_STATE_VERSION; + state.sequence = states.have_newest + ? states.state[states.newest_slot].sequence + 1 + : 1; + state.clean_shutdown = clean_shutdown; + state.checksum = rc_allocator_clean_state_checksum(&state); + return rc_allocator_write_clean_state(al, target_slot, &state, TRUE); +} + +static platform_status +rc_allocator_initialize_clean_states(rc_allocator *al) +{ + for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { + rc_allocator_clean_state state; + ZERO_CONTENTS(&state); + state.magic = RC_ALLOCATOR_CLEAN_STATE_MAGIC; + state.format_version = RC_ALLOCATOR_CLEAN_STATE_VERSION; + state.sequence = slot + 1; + state.clean_shutdown = FALSE; + state.checksum = rc_allocator_clean_state_checksum(&state); + + platform_status rc = + rc_allocator_write_clean_state(al, slot, &state, FALSE); + if (!SUCCESS(rc)) { + return rc; + } + } + return io_durable_barrier(al->io); +} + +static void +rc_allocator_record_allocated_extent(rc_allocator *al) +{ + int64 curr_allocated = __sync_add_and_fetch(&al->stats.curr_allocated, 1); + int64 max_allocated = al->stats.max_allocated; + while (curr_allocated > max_allocated) { + __sync_bool_compare_and_swap( + &al->stats.max_allocated, max_allocated, curr_allocated); + max_allocated = al->stats.max_allocated; + } +} + +static platform_status +rc_allocator_recovery_initialize_refcounts(rc_allocator *al) +{ + uint64 reserved_extent_count = rc_allocator_reserved_extent_count(al->cfg); + + if (reserved_extent_count > al->cfg->extent_capacity) { + platform_error_log("Allocator needs %lu reserved extents, but its " + "configured capacity is only %lu extents.\n", + reserved_extent_count, + al->cfg->extent_capacity); + return STATUS_BAD_PARAM; + } + + memset(al->ref_count, + 0, + rc_allocator_refcount_buffer_size(al->cfg)); + + /* + * Extent 0 contains both the allocator meta page and every fixed table + * superblock. The refcount table begins at extent 1; the two extents + * after it hold alternating clean-state records. + */ + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) + { + platform_assert(al->ref_count[extent_no] == AL_FREE); + al->ref_count[extent_no] = AL_ONE_REF; + rc_allocator_record_allocated_extent(al); + } + + /* Match a regular mount: the first post-recovery allocation scans at 0. */ + al->hand = 0; + return STATUS_OK; +} + +static bool32 +rc_allocator_recovery_extent_is_reserved(const rc_allocator *al, + uint64 extent_no) +{ + return extent_no < rc_allocator_reserved_extent_count(al->cfg); +} + static platform_status rc_allocator_validate_disk_geometry(rc_allocator *al) { @@ -309,7 +603,9 @@ rc_allocator_init_meta_page(rc_allocator *al) memset(al->meta_page->splinters, INVALID_ALLOCATOR_ROOT_ID, sizeof(al->meta_page->splinters)); - al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); + al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); + al->meta_page->format_magic = RC_ALLOCATOR_FORMAT_MAGIC; + al->meta_page->format_version = RC_ALLOCATOR_FORMAT_VERSION; return STATUS_OK; } @@ -344,6 +640,15 @@ rc_allocator_valid_config(allocator_config *cfg) return STATUS_BAD_PARAM; } + if (rc_allocator_reserved_extent_count(cfg) > cfg->extent_capacity) { + platform_error_log("Configured allocator has %lu extents, but needs %lu " + "reserved extents for metadata, refcounts, and clean " + "state.\n", + cfg->extent_capacity, + rc_allocator_reserved_extent_count(cfg)); + return STATUS_BAD_PARAM; + } + // Assert: Disk size == (page-size * #-of-pages) if (cfg->capacity != (cfg->io_cfg->page_size * cfg->page_capacity)) { platform_error_log("Configured disk size, %lu bytes, is not an integral" @@ -410,8 +715,7 @@ rc_allocator_init(rc_allocator *al, return rc; } // To ensure alignment always allocate in multiples of page size. - uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); - buffer_size = ROUNDUP(buffer_size, cfg->io_cfg->page_size); + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); rc = platform_buffer_init(&al->bh, buffer_size); if (!SUCCESS(rc)) { platform_mutex_destroy(&al->lock); @@ -431,14 +735,42 @@ rc_allocator_init(rc_allocator *al, * Allocate room for the ref counts, use same rounded up size used in buffer * creation. */ - rc_extent_count = (buffer_size + al->cfg->io_cfg->extent_size - 1) - / al->cfg->io_cfg->extent_size; + rc_extent_count = rc_allocator_refcount_extent_count(cfg); for (uint64 i = 0; i < rc_extent_count; i++) { allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); platform_assert(addr == cfg->io_cfg->extent_size * (i + 1)); } + for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { + allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); + platform_assert(addr == rc_allocator_clean_state_addr(cfg, slot)); + } + + /* + * Persist the immutable bootstrap layout and both initial false state + * records before returning a newly created allocator. A crash before a + * later clean close therefore enters rebuild recovery rather than trusting + * this freshly initialized refcount map. + */ + rc = rc_allocator_initialize_clean_states(al); + if (!SUCCESS(rc)) { + goto deinit_allocator; + } + rc = rc_allocator_write_meta_page(al); + if (!SUCCESS(rc)) { + goto deinit_allocator; + } + rc = io_durable_barrier(al->io); + if (!SUCCESS(rc)) { + goto deinit_allocator; + } + return STATUS_OK; + +deinit_allocator: + rc_allocator_deinit(al); + ZERO_CONTENTS(al); + return rc; } void @@ -458,12 +790,13 @@ rc_allocator_deinit(rc_allocator *al) * Write the file system to disk *---------------------------------------------------------------------- */ -platform_status -rc_allocator_mount(rc_allocator *al, - allocator_config *cfg, - io_handle *io, - platform_heap_id hid, - platform_module_id mid) +static platform_status +rc_allocator_mount_internal(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid, + bool32 rebuild_refcounts) { platform_status status; @@ -493,8 +826,7 @@ rc_allocator_mount(rc_allocator *al, platform_assert(cfg->capacity == cfg->io_cfg->page_size * cfg->page_capacity); - uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); - buffer_size = ROUNDUP(buffer_size, cfg->io_cfg->page_size); + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); status = platform_buffer_init(&al->bh, buffer_size); if (!SUCCESS(status)) { platform_free(al->heap_id, al->meta_page); @@ -508,45 +840,179 @@ rc_allocator_mount(rc_allocator *al, status = io_read( io, al->meta_page, al->cfg->io_cfg->page_size, RC_ALLOCATOR_BASE_OFFSET); if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return status; + goto deinit_buffer; } status = rc_allocator_validate_disk_geometry(al); if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return status; + goto deinit_buffer; } // validate the checksum of the meta page. checksum128 currChecksum = rc_allocator_meta_page_checksum(al->meta_page); if (!platform_checksum_is_equal(al->meta_page->checksum, currChecksum)) { platform_error_log("Corrupt SplinterDB allocator meta page on mount\n"); - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return STATUS_BAD_PARAM; + status = STATUS_BAD_PARAM; + goto deinit_buffer; } - // load the ref counts from disk. - status = io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); - if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return status; + if (al->meta_page->format_magic != RC_ALLOCATOR_FORMAT_MAGIC + || al->meta_page->format_version != RC_ALLOCATOR_FORMAT_VERSION) + { + platform_error_log("Unsupported SplinterDB allocator bootstrap format " + "on mount.\n"); + status = STATUS_BAD_PARAM; + goto deinit_buffer; + } + + if (!rebuild_refcounts) { + rc_allocator_clean_states states; + status = rc_allocator_load_clean_states(al, &states); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + if (!states.have_newest || states.duplicate_sequence + || !states.state[states.newest_slot].clean_shutdown) + { + platform_error_log("Allocator was not cleanly shut down; recovery " + "rebuild is required.\n"); + status = STATUS_INVALID_STATE; + goto deinit_buffer; + } } - for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { - if (al->ref_count[i] != 0) { - al->stats.curr_allocated++; + if (rebuild_refcounts) { + /* Do not leave a stale clean state if recovery itself is interrupted. */ + status = rc_allocator_publish_clean_state(al, FALSE); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + status = rc_allocator_recovery_initialize_refcounts(al); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + al->recovery_in_progress = TRUE; + } else { + // Load the ref counts from disk during a normal, clean mount. + status = io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + + for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { + if (al->ref_count[i] != 0) { + al->stats.curr_allocated++; + } + } + + /* Mark dirty before handing the trusted map to any mutating caller. */ + status = rc_allocator_publish_clean_state(al, FALSE); + if (!SUCCESS(status)) { + goto deinit_buffer; } } return STATUS_OK; + +deinit_buffer: + platform_buffer_deinit(&al->bh); + al->ref_count = NULL; + platform_free(al->heap_id, al->meta_page); + al->meta_page = NULL; + platform_mutex_destroy(&al->lock); + return status; +} + +platform_status +rc_allocator_mount(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid) +{ + return rc_allocator_mount_internal(al, cfg, io, hid, mid, FALSE); +} + +platform_status +rc_allocator_mount_recovery(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid) +{ + return rc_allocator_mount_internal(al, cfg, io, hid, mid, TRUE); +} + +platform_status +rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr) +{ + if (!al->recovery_in_progress) { + platform_error_log("Cannot acquire allocator extent while recovery is " + "not in progress.\n"); + return STATUS_INVALID_STATE; + } + + uint64 extent_size = al->cfg->io_cfg->extent_size; + if (extent_addr >= al->cfg->capacity || extent_addr % extent_size != 0) { + platform_error_log("Invalid allocator recovery extent address %lu.\n", + extent_addr); + return STATUS_BAD_PARAM; + } + + uint64 extent_no = rc_allocator_extent_number(al, extent_addr); + platform_assert(extent_no < al->cfg->extent_capacity); + if (rc_allocator_recovery_extent_is_reserved(al, extent_no)) { + platform_error_log("Cannot rebuild ownership of reserved allocator " + "extent %lu.\n", + extent_no); + return STATUS_BAD_PARAM; + } + + while (TRUE) { + refcount old_ref = __atomic_load_n(&al->ref_count[extent_no], + __ATOMIC_RELAXED); + if (old_ref == (refcount)-1) { + platform_error_log("Allocator recovery refcount overflow for extent " + "%lu.\n", + extent_no); + return STATUS_LIMIT_EXCEEDED; + } + + refcount new_ref = + old_ref == AL_FREE ? AL_ONE_REF : old_ref + 1; + if (!__sync_bool_compare_and_swap( + &al->ref_count[extent_no], old_ref, new_ref)) + { + continue; + } + + if (old_ref == AL_FREE) { + rc_allocator_record_allocated_extent(al); + } + return STATUS_OK; + } +} + +void +rc_allocator_rebuild_finish(rc_allocator *al) +{ + platform_assert(al != NULL); + platform_assert(al->recovery_in_progress); + + /* + * Intentionally no I/O here. A rebuilt map is durable only after a later + * clean unmount; another crash before then simply rebuilds it again. + */ + al->recovery_in_progress = FALSE; +} + +void +rc_allocator_abort_recovery(rc_allocator *al) +{ + platform_assert(al != NULL); + platform_assert(al->recovery_in_progress); + + rc_allocator_deinit(al); + ZERO_CONTENTS(al); } @@ -555,12 +1021,32 @@ rc_allocator_unmount(rc_allocator *al) { platform_status status; + if (al->recovery_in_progress) { + platform_error_log("Discarding incomplete allocator recovery instead of " + "persisting its partial refcount map.\n"); + rc_allocator_abort_recovery(al); + return; + } + // persist the ref counts upon unmount. - uint64 buffer_size = al->cfg->extent_capacity * sizeof(refcount); + uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); uint32 io_size = ROUNDUP(buffer_size, al->cfg->io_cfg->page_size); status = io_write(al->io, al->ref_count, io_size, al->cfg->io_cfg->extent_size); platform_assert_status_ok(status); + + /* + * This is the sole normal persistence point for the allocator map. The + * checkpoint record was already made durable by core_unmount(); do not + * advertise a clean allocator snapshot until this write has crossed the + * device durability boundary as well. + */ + status = io_durable_barrier(al->io); + platform_assert_status_ok(status); + + /* Publish clean permission in the alternate durable state record. */ + status = rc_allocator_publish_clean_state(al, TRUE); + platform_assert_status_ok(status); rc_allocator_deinit(al); } @@ -686,12 +1172,7 @@ rc_allocator_alloc_super_addr(rc_allocator *al, // assign the first available slot and update the on disk metadata. al->meta_page->splinters[idx] = allocator_root_id; *addr = (1 + idx) * al->cfg->io_cfg->page_size; - al->meta_page->checksum = - rc_allocator_meta_page_checksum(al->meta_page); - platform_status io_status = io_write(al->io, - al->meta_page, - al->cfg->io_cfg->page_size, - RC_ALLOCATOR_BASE_OFFSET); + platform_status io_status = rc_allocator_write_meta_page(al); platform_assert_status_ok(io_status); status = STATUS_OK; break; @@ -715,12 +1196,7 @@ rc_allocator_remove_super_addr(rc_allocator *al, */ if (al->meta_page->splinters[idx] == allocator_root_id) { al->meta_page->splinters[idx] = INVALID_ALLOCATOR_ROOT_ID; - al->meta_page->checksum = - rc_allocator_meta_page_checksum(al->meta_page); - platform_status status = io_write(al->io, - al->meta_page, - al->cfg->io_cfg->page_size, - RC_ALLOCATOR_BASE_OFFSET); + platform_status status = rc_allocator_write_meta_page(al); platform_assert_status_ok(status); platform_mutex_unlock(&al->lock); return; diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 06ed72562..079e565a6 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -33,7 +33,14 @@ *---------------------------------------------------------------------- */ typedef struct ONDISK rc_allocator_meta_page { - disk_geometry geometry; + disk_geometry geometry; + /* + * Identifies the immutable bootstrap layout. In particular, it proves + * that the two fixed clean-state extents after the refcount map are owned + * by this allocator rather than by an older on-disk format. + */ + uint64 format_magic; + uint64 format_version; allocator_root_id splinters[RC_ALLOCATOR_MAX_ROOT_IDS]; checksum128 checksum; } rc_allocator_meta_page; @@ -76,6 +83,13 @@ typedef struct rc_allocator { platform_mutex lock; platform_heap_id heap_id; + /* + * True between rc_allocator_mount_recovery() and either + * rc_allocator_rebuild_finish() or rc_allocator_abort_recovery(). An + * incomplete rebuilt map must never be written back to disk. + */ + bool32 recovery_in_progress; + // Stats -- not distributed for now rc_allocator_stats stats; } rc_allocator; @@ -90,6 +104,14 @@ rc_allocator_init(rc_allocator *al, void rc_allocator_deinit(rc_allocator *al); +/* + * Normal mount accepts only a durable clean-state record and publishes an + * unclean state record before returning. Those records occupy two fixed, + * alternating extents after the persisted refcount map, so normal lifecycle + * operations never rewrite the sole allocator bootstrap page. + * STATUS_INVALID_STATE means callers must use the recovery-rebuild path + * instead of trusting the persisted refcount map. + */ platform_status rc_allocator_mount(rc_allocator *al, allocator_config *cfg, @@ -97,6 +119,43 @@ rc_allocator_mount(rc_allocator *al, platform_heap_id hid, platform_module_id mid); +/* + * Mount the allocator for crash recovery. This validates and retains the + * allocator metadata page, but deliberately ignores the persisted refcount + * table. The caller must rebuild the in-memory table from durable objects, + * then call rc_allocator_rebuild_finish() before using normal allocator + * lifecycle operations. + */ +platform_status +rc_allocator_mount_recovery(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid); + +/* + * Add one logical ownership reference for an extent while rebuilding a + * recovery map. The first reference establishes the allocator's nonzero + * allocation floor (AL_ONE_REF); later references increment it normally. + * extent_addr must be the base address of a non-reserved allocator extent. + */ +platform_status +rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr); + +/* + * Complete a successful rebuild without performing I/O. A later normal + * rc_allocator_unmount() may persist the rebuilt table on clean shutdown. + */ +void +rc_allocator_rebuild_finish(rc_allocator *al); + +/* + * Discard a partially rebuilt recovery map. Unlike rc_allocator_unmount(), + * this never writes allocator state to disk. + */ +void +rc_allocator_abort_recovery(rc_allocator *al); + platform_status rc_allocator_read_disk_geometry(const char *filename, disk_geometry *geometry); diff --git a/src/shard_log.c b/src/shard_log.c index c0eee1319..9c1fd0162 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -25,8 +25,19 @@ static uint64 shard_log_magic_idx = 0; -int -shard_log_write(log_handle *log, key tuple_key, message msg, uint64 generation); +int shard_log_write(log_handle *log, + key tuple_key, + message msg, + uint64 memtable_generation, + uint64 leaf_generation); +platform_status +shard_log_seal(log_handle *log); +platform_status +shard_log_rotate(log_handle *log, + log_segment_info *sealed, + log_segment_info *fresh); +void +shard_log_release(log_handle *log); uint64 shard_log_addr(log_handle *log); uint64 @@ -36,6 +47,9 @@ shard_log_magic(log_handle *log); static log_ops shard_log_ops = { .write = shard_log_write, + .seal = shard_log_seal, + .rotate = shard_log_rotate, + .release = shard_log_release, .addr = shard_log_addr, .meta_addr = shard_log_meta_addr, .magic = shard_log_magic, @@ -93,6 +107,9 @@ page_handle * shard_log_alloc(shard_log *log, uint64 *next_extent) { uint64 addr = mini_alloc_page(&log->mini, 0, next_extent); + if (addr == 0) { + return NULL; + } return cache_alloc(log->cc, addr, PAGE_TYPE_LOG); } @@ -142,7 +159,15 @@ shard_log_zap(shard_log *log) thread_data->offset = 0; } - mini_dec_ref(cc, log->meta_head, PAGE_TYPE_LOG); + if (log->meta_head != 0) { + /* Drop unused per-batch reserves before releasing the mini root. */ + mini_release(&log->mini); + refcount ref = mini_dec_ref(cc, log->meta_head, PAGE_TYPE_LOG); + platform_assert(ref == 0); + log->meta_head = 0; + log->addr = 0; + log->has_pages = FALSE; + } } /* @@ -152,11 +177,12 @@ shard_log_zap(shard_log *log) * ------------------------------------------------------------------------- */ struct ONDISK log_entry { - uint64 generation; + uint64 memtable_generation; + uint64 leaf_generation; ondisk_tuple tuple; }; -#define INVALID_GENERATION ((uint64) - 1) +#define INVALID_LOG_GENERATION ((uint64)-1) static key log_entry_key(log_entry *le) @@ -200,7 +226,19 @@ static bool32 terminal_log_entry(shard_log_config *cfg, char *page, log_entry *le) { return page + shard_log_page_size(cfg) - (char *)le < sizeof(log_entry) - || le->generation == INVALID_GENERATION; + || le->memtable_generation == INVALID_LOG_GENERATION; +} + +static inline void +log_entry_set_terminal(log_entry *le) +{ + /* + * terminal_log_entry() tests memtable_generation. Mark both generation + * fields invalid so the terminator cannot be confused with a record by + * diagnostics or a future format validator. + */ + le->leaf_generation = INVALID_LOG_GENERATION; + le->memtable_generation = INVALID_LOG_GENERATION; } static log_entry * @@ -217,19 +255,29 @@ get_new_page_for_thread(shard_log *log, uint64 next_extent; *page = shard_log_alloc(log, &next_extent); + if (*page == NULL) { + return -1; + } thread_data->addr = (*page)->disk_addr; shard_log_hdr *hdr = (shard_log_hdr *)(*page)->data; hdr->magic = log->magic; hdr->next_extent_addr = next_extent; hdr->num_entries = 0; thread_data->offset = sizeof(shard_log_hdr); + log->has_pages = TRUE; return 0; } int -shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) +shard_log_write(log_handle *logh, + key tuple_key, + message msg, + uint64 memtable_generation, + uint64 leaf_generation) { debug_assert(key_is_user_key(tuple_key)); + debug_assert(memtable_generation != INVALID_LOG_GENERATION); + debug_assert(leaf_generation != INVALID_LOG_GENERATION); shard_log *log = (shard_log *)logh; cache *cc = log->cc; @@ -290,7 +338,7 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) if (free_space < new_entry_size) { if (sizeof(log_entry) <= free_space) { - cursor->generation = INVALID_GENERATION; + log_entry_set_terminal(cursor); } hdr->checksum = shard_log_checksum(log->cfg, page); @@ -309,7 +357,8 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) hdr = (shard_log_hdr *)page->data; } - cursor->generation = generation; + cursor->memtable_generation = memtable_generation; + cursor->leaf_generation = leaf_generation; copy_tuple_to_ondisk_tuple(&cursor->tuple, tuple_key, msg); hdr->num_entries++; @@ -332,6 +381,152 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) return 0; } +/* + * shard_log_seal -- + * + * Finalize every currently active per-thread append page. This is + * deliberately bounded by MAX_THREADS: it does not walk the historical + * log. A final terminal record (where there is room) and checksum make + * each page readable by shard_log_iterator_init(), then clearing the + * append cursor ensures a subsequent writer allocates a new page instead + * of changing the sealed one. + * + * The caller must prevent concurrent shard_log_write() and seal calls. + * In particular, thread_data is otherwise only protected by the + * per-thread writer convention, not by a log-wide lock. This function + * intentionally does not issue writeback: after establishing that + * exclusion, the caller takes cache_writeback_fence(), followed by a + * durable barrier. + */ +platform_status +shard_log_seal(log_handle *logh) +{ + shard_log *log = (shard_log *)logh; + cache *cc = log->cc; + + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(log, thr_i); + uint64 addr = thread_data->addr; + if (addr == SHARD_UNMAPPED) { + continue; + } + + page_handle *page = cache_get(cc, addr, TRUE, PAGE_TYPE_LOG); + uint64 wait = 1; + while (!cache_try_claim(cc, page)) { + cache_unget(cc, page); + platform_sleep_ns(wait); + wait = wait > 1024 ? wait : 2 * wait; + page = cache_get(cc, addr, TRUE, PAGE_TYPE_LOG); + } + cache_lock(cc, page); + + debug_assert(thread_data->addr == addr); + debug_assert(thread_data->offset >= sizeof(shard_log_hdr)); + debug_assert(thread_data->offset <= shard_log_page_size(log->cfg)); + + shard_log_hdr *hdr = (shard_log_hdr *)page->data; + log_entry *cursor = + (log_entry *)(page->data + thread_data->offset); + uint64 free_space = shard_log_page_size(log->cfg) - thread_data->offset; + if (sizeof(log_entry) <= free_space) { + log_entry_set_terminal(cursor); + } + hdr->checksum = shard_log_checksum(log->cfg, page); + cache_mark_dirty(cc, page); + + cache_unlock(cc, page); + cache_unclaim(cc, page); + cache_unget(cc, page); + + /* Subsequent writes must allocate a new append page. */ + thread_data->addr = SHARD_UNMAPPED; + thread_data->offset = 0; + } + + return STATUS_OK; +} + +/* + * Detach a sealed stream from the live log. The original allocation reference + * on its metadata extent is transferred to sealed; an eventual durable + * descriptor must own that reference. A fresh mini allocator is prepared + * before the old stream is sealed, so core can publish the fresh identity + * while inserts remain excluded. This only prepares a physical boundary; it + * does not publish or advance the logical durable-log tail. + */ +platform_status +shard_log_rotate(log_handle *logh, + log_segment_info *sealed, + log_segment_info *fresh_info) +{ + shard_log *log = (shard_log *)logh; + shard_log fresh; + + platform_assert(sealed != NULL); + platform_assert(fresh_info != NULL); + ZERO_CONTENTS(sealed); + ZERO_CONTENTS(fresh_info); + + platform_status rc = shard_log_init(&fresh, log->cc, log->cfg); + if (!SUCCESS(rc)) { + return rc; + } + + rc = shard_log_seal(logh); + if (!SUCCESS(rc)) { + shard_log_zap(&fresh); + return rc; + } + + /* No future allocation may use the old mini allocator. */ + mini_release(&log->mini); + + if (log->has_pages) { + *sealed = (log_segment_info){ + .addr = log->addr, + .meta_addr = log->meta_head, + .magic = log->magic, + }; + } else { + /* An empty segment has no descriptor and no retained ownership. */ + refcount ref = mini_dec_ref(log->cc, log->meta_head, PAGE_TYPE_LOG); + platform_assert(ref == 0); + } + + log->mini = fresh.mini; + log->addr = fresh.addr; + log->meta_head = fresh.meta_head; + log->magic = fresh.magic; + log->has_pages = FALSE; + memcpy(log->thread_data, fresh.thread_data, sizeof(log->thread_data)); + + *fresh_info = (log_segment_info){ + .addr = log->addr, + .meta_addr = log->meta_head, + .magic = log->magic, + }; + return STATUS_OK; +} + +void +shard_log_segment_discard(cache *cc, + const log_segment_info *segment) +{ + if (segment->meta_addr == 0) { + return; + } + refcount ref = mini_dec_ref(cc, segment->meta_addr, PAGE_TYPE_LOG); + platform_assert(ref == 0); +} + +void +shard_log_release(log_handle *logh) +{ + shard_log_zap((shard_log *)logh); +} + uint64 shard_log_addr(log_handle *logh) { @@ -374,7 +569,20 @@ shard_log_compare(const void *p1, const void *p2, void *unused) { log_entry **le1 = (log_entry **)p1; log_entry **le2 = (log_entry **)p2; - return (*le1)->generation - (*le2)->generation; + + if ((*le1)->memtable_generation < (*le2)->memtable_generation) { + return -1; + } + if ((*le1)->memtable_generation > (*le2)->memtable_generation) { + return 1; + } + if ((*le1)->leaf_generation < (*le2)->leaf_generation) { + return -1; + } + if ((*le1)->leaf_generation > (*le2)->leaf_generation) { + return 1; + } + return 0; } log_handle * @@ -527,6 +735,16 @@ shard_log_iterator_curr(iterator *itorh, key *curr_key, message *msg) *msg = log_entry_message(itor->cc, itor->entries[itor->pos]); } +void +shard_log_iterator_curr_generations(shard_log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation) +{ + platform_assert(itor->pos < itor->num_entries); + *memtable_generation = itor->entries[itor->pos]->memtable_generation; + *leaf_generation = itor->entries[itor->pos]->leaf_generation; +} + bool32 shard_log_iterator_can_prev(iterator *itorh) { @@ -597,11 +815,12 @@ shard_log_print(shard_log *log) le = log_entry_next(le)) { platform_default_log( - "%s -- %s%s : %lu\n", + "%s -- %s%s : memtable=%lu leaf=%lu\n", key_string(dcfg, log_entry_key(le)), log_entry_message_is_blob(le) ? "(blob) " : "", message_string(dcfg, log_entry_message(cc, le)), - le->generation); + le->memtable_generation, + le->leaf_generation); } } cache_unget(cc, page); diff --git a/src/shard_log.h b/src/shard_log.h index 6459d78d9..3e024d146 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -46,6 +46,8 @@ typedef struct shard_log { uint64 addr; uint64 meta_head; uint64 magic; + /* Set once any log page has been allocated; survives sealing. */ + bool32 has_pages; } shard_log; typedef struct log_entry log_entry; @@ -76,9 +78,23 @@ typedef struct ONDISK shard_log_hdr { platform_status shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg); +platform_status +shard_log_rotate(log_handle *log, + log_segment_info *sealed, + log_segment_info *fresh); + void shard_log_zap(shard_log *log); +/* + * Drop the external mini-allocator ownership transferred to a detached + * segment. It is for failed rotation cleanup and tests; persisted segment + * descriptors will eventually own and release this reference instead. + */ +void +shard_log_segment_discard(cache *cc, + const log_segment_info *segment); + platform_status shard_log_iterator_init(cache *cc, shard_log_config *cfg, @@ -90,6 +106,15 @@ shard_log_iterator_init(cache *cc, void shard_log_iterator_deinit(platform_heap_id hid, shard_log_iterator *itor); +/* + * Return the generation metadata of the current record. The caller must + * first establish that the iterator has a current record. + */ +void +shard_log_iterator_curr_generations(shard_log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation); + void shard_log_config_init(shard_log_config *log_cfg, cache_config *cache_cfg, diff --git a/src/splinterdb.c b/src/splinterdb.c index b86146685..c40dc8c36 100644 --- a/src/splinterdb.c +++ b/src/splinterdb.c @@ -498,7 +498,8 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN deinit_cache: clockcache_deinit(&kvs->cache_handle); deinit_allocator: - rc_allocator_unmount(&kvs->allocator_handle); + /* Initialization/open failed before a successful clean core shutdown. */ + rc_allocator_deinit(&kvs->allocator_handle); deinit_system: task_system_deinit(&kvs->task_sys); deinit_iohandle: @@ -578,7 +579,13 @@ splinterdb_close(splinterdb **kvs_in) // IN } io_wait_all(kvs->io_handle); clockcache_deinit(&kvs->cache_handle); - rc_allocator_unmount(&kvs->allocator_handle); + if (SUCCESS(status)) { + /* This writes the map and only then publishes allocator clean state. */ + rc_allocator_unmount(&kvs->allocator_handle); + } else { + /* Keep the durable clean marker cleared; the next open must rebuild. */ + rc_allocator_deinit(&kvs->allocator_handle); + } task_system_deinit(&kvs->task_sys); io_handle_destroy(kvs->io_handle); diff --git a/src/trunk.c b/src/trunk.c index 133139a5c..d24c17ae5 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -2541,6 +2541,46 @@ trunk_read_end(trunk_context *context) batch_rwlock_unget(&context->root_lock, 0); } +/* + * Capture the root address and acquire its allocator reference while the root + * lock prevents COW publication from dropping the live reference. This is + * deliberately metadata-only: checkpointing must not fault the root page just + * to take a snapshot. + */ +platform_status +trunk_snapshot_acquire(trunk_context *context, trunk_snapshot *snapshot) +{ + snapshot->root_addr = 0; + + trunk_read_begin(context); + if (context->root != NULL) { + snapshot->root_addr = context->root->ref.addr; + trunk_inc_ref(context->al, snapshot->root_addr); + } + trunk_read_end(context); + + return STATUS_OK; +} + +platform_status +trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot) +{ + if (snapshot->root_addr == 0) { + return STATUS_OK; + } + + platform_status rc = trunk_dec_ref(context->cfg, + context->hid, + context->cc, + context->al, + context->ts, + snapshot->root_addr); + if (SUCCESS(rc)) { + snapshot->root_addr = 0; + } + return rc; +} + platform_status trunk_init_root_handle(trunk_context *context, trunk_ondisk_node_handle *handle) { @@ -6814,8 +6854,11 @@ trunk_context_clone(trunk_context *dst, trunk_context *src) platform_status trunk_make_durable(trunk_context *context) { - cache_flush(context->cc); - return STATUS_OK; + platform_status rc = cache_writeback_fence(context->cc); + if (!SUCCESS(rc)) { + return rc; + } + return cache_durable_barrier(context->cc); } /************************************ diff --git a/src/trunk.h b/src/trunk.h index 7df20d094..a48dbab07 100644 --- a/src/trunk.h +++ b/src/trunk.h @@ -174,6 +174,15 @@ typedef struct trunk_context { incorporation_tasks tasks; } trunk_context; +/* + * An owned, point-in-time reference to a COW trunk root. The reference keeps + * the root (and therefore its reachable subtree) alive until it is either + * released or transferred to a durable checkpoint record. + */ +typedef struct trunk_snapshot { + uint64 root_addr; +} trunk_snapshot; + typedef struct trunk_ondisk_node_handle { cache *cc; page_handle *header_page; @@ -223,6 +232,14 @@ trunk_context_deinit(trunk_context *context); platform_status trunk_context_clone(trunk_context *dst, trunk_context *src); +/* Capture an owned reference to the current COW root without reading it. */ +platform_status +trunk_snapshot_acquire(trunk_context *context, trunk_snapshot *snapshot); + +/* Drop an owned snapshot reference that was not published. */ +platform_status +trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot); + /* Make a trunk durable */ platform_status trunk_make_durable(trunk_context *context); diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 2511e6152..139db99e5 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -17,6 +17,7 @@ #include "rc_allocator.h" #include "cache.h" #include "clockcache.h" +#include "mini_allocator.h" #include "random.h" #include "task.h" @@ -70,6 +71,198 @@ test_btree_process_noop(void *arg, uint64 generation) // really a no-op } +static platform_status +test_memtable_generation_init(cache *cc, + test_btree_config *cfg, + platform_heap_id hid) +{ + const uint64 first_generation = 17; + const uint64 max_memtables = 4; + memtable_config mt_cfg = *cfg->mt_cfg; + memtable_context mt_ctxt; + + mt_cfg.max_memtables = max_memtables; + + platform_status rc = memtable_context_init( + &mt_ctxt, hid, cc, &mt_cfg, test_btree_process_noop, NULL); + if (!SUCCESS(rc)) { + return rc; + } + + if (mt_ctxt.generation != 0 || mt_ctxt.generation_to_incorporate != 0 + || mt_ctxt.generation_retired != (uint64)-1) + { + platform_error_log("memtable fresh initialization has unexpected " + "generation state\n"); + rc = STATUS_TEST_FAILED; + goto deinit_fresh; + } + + for (uint64 generation = 0; generation < max_memtables; generation++) { + uint64 mt_no = generation % max_memtables; + if (mt_ctxt.mt[mt_no].generation != generation) { + platform_error_log("memtable fresh initialization put generation " + "%lu in the wrong slot\n", + generation); + rc = STATUS_TEST_FAILED; + goto deinit_fresh; + } + } + +deinit_fresh: + memtable_context_deinit(&mt_ctxt); + if (!SUCCESS(rc)) { + return rc; + } + + rc = memtable_context_init_at_generation(&mt_ctxt, + hid, + cc, + &mt_cfg, + test_btree_process_noop, + NULL, + first_generation); + if (!SUCCESS(rc)) { + return rc; + } + + if (mt_ctxt.generation != first_generation + || mt_ctxt.generation_to_incorporate != first_generation + || mt_ctxt.generation_retired != first_generation - 1) + { + platform_error_log("memtable recovery initialization has unexpected " + "generation state\n"); + rc = STATUS_TEST_FAILED; + goto deinit_recovery; + } + + for (uint64 generation = first_generation; + generation < first_generation + max_memtables; + generation++) + { + uint64 mt_no = generation % max_memtables; + if (mt_ctxt.mt[mt_no].generation != generation) { + platform_error_log("memtable recovery initialization put generation " + "%lu in the wrong slot\n", + generation); + rc = STATUS_TEST_FAILED; + goto deinit_recovery; + } + } + + memtable_block_inserts(&mt_ctxt); + memtable_unblock_inserts(&mt_ctxt); + +deinit_recovery: + memtable_context_deinit(&mt_ctxt); + if (SUCCESS(rc)) { + platform_default_log("btree_test: memtable generation init test passed\n"); + } + return rc; +} + +typedef struct test_mini_recovery_walk_state { + uint64 metadata_visits; + uint64 data_visits; + uint64 metadata_extent; + uint64 data_extent; + bool32 fail_data_visit; +} test_mini_recovery_walk_state; + +static platform_status +test_mini_recovery_walk_visit(uint64 extent_addr, + page_type type, + mini_recovery_extent_kind kind, + uint64 batch, + void *arg) +{ + test_mini_recovery_walk_state *state = arg; + + if (type != PAGE_TYPE_MISC) { + return STATUS_TEST_FAILED; + } + + if (kind == MINI_RECOVERY_EXTENT_METADATA) { + if (batch != MINI_RECOVERY_METADATA_BATCH) { + return STATUS_TEST_FAILED; + } + state->metadata_visits++; + state->metadata_extent = extent_addr; + return STATUS_OK; + } + + if (kind != MINI_RECOVERY_EXTENT_DATA || batch != 0) { + return STATUS_TEST_FAILED; + } + + state->data_visits++; + state->data_extent = extent_addr; + return state->fail_data_visit ? STATUS_TEST_FAILED : STATUS_OK; +} + +static platform_status +test_mini_recovery_walk(cache *cc) +{ + allocator *al = cache_get_allocator(cc); + mini_allocator mini; + test_mini_recovery_walk_state state; + uint64 meta_head = 0; + uint64 data_extent = 0; + platform_status rc = + allocator_alloc(al, &meta_head, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + return rc; + } + + mini_init(&mini, cc, meta_head, 0, 1, PAGE_TYPE_MISC); + data_extent = mini_alloc_extent(&mini, 0, NULL); + if (data_extent == 0) { + rc = STATUS_NO_SPACE; + goto cleanup; + } + + ZERO_CONTENTS(&state); + rc = mini_recovery_walk( + cc, meta_head, PAGE_TYPE_MISC, test_mini_recovery_walk_visit, &state); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (state.metadata_visits != 1 || state.data_visits != 1 + || state.metadata_extent != meta_head + || state.data_extent != data_extent) + { + platform_error_log("mini recovery walker returned unexpected extents\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + ZERO_CONTENTS(&state); + state.fail_data_visit = TRUE; + rc = mini_recovery_walk( + cc, meta_head, PAGE_TYPE_MISC, test_mini_recovery_walk_visit, &state); + if (!STATUS_IS_EQ(rc, STATUS_TEST_FAILED) || state.metadata_visits != 1 + || state.data_visits != 1) + { + platform_error_log("mini recovery walker did not propagate callback " + "failure\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = STATUS_OK; + +cleanup: + mini_release(&mini); + refcount ref = mini_dec_ref(cc, meta_head, PAGE_TYPE_MISC); + if (ref != 0 && SUCCESS(rc)) { + platform_error_log("mini recovery walker left an unexpected mini ref\n"); + rc = STATUS_TEST_FAILED; + } + if (SUCCESS(rc)) { + platform_default_log("btree_test: mini recovery walker test passed\n"); + } + return rc; +} + test_memtable_context * test_memtable_context_create(cache *cc, test_btree_config *cfg, @@ -2258,6 +2451,12 @@ btree_test(int argc, char *argv[]) platform_assert_status_ok(rc); cache *ccp = (cache *)cc; + rc = test_memtable_generation_init(ccp, &test_cfg, hid); + platform_assert_status_ok(rc); + + rc = test_mini_recovery_walk(ccp); + platform_assert_status_ok(rc); + uint64 max_tuples_per_memtable = test_cfg.mt_cfg->max_extents_per_memtable * cache_config_extent_size((cache_config *)&system_cfg.cache_cfg) / 3 diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index ad4b3b273..d75267d37 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -90,6 +90,688 @@ cache_test_alloc_extents(cache *cc, return rc; } +static platform_status +cache_test_fill_page(cache *cc, uint64 addr, uint64 page_size, uint8 value) +{ + page_handle *page = cache_get(cc, addr, TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, page)) { + cache_unget(cc, page); + return STATUS_BUSY; + } + + cache_lock(cc, page); + memset(page->data, value, page_size); + cache_unlock(cc, page); + cache_unclaim(cc, page); + cache_unget(cc, page); + return STATUS_OK; +} + +static platform_status +cache_test_verify_disk_page(cache *cc, + uint64 addr, + uint64 page_size, + uint8 expected) +{ + clockcache *clock = (clockcache *)cc; + buffer_handle buffer; + platform_status rc = platform_buffer_init(&buffer, page_size); + if (!SUCCESS(rc)) { + return rc; + } + + rc = io_read(clock->io, platform_buffer_getaddr(&buffer), page_size, addr); + if (SUCCESS(rc)) { + const uint8 *bytes = platform_buffer_getaddr(&buffer); + for (uint64 i = 0; i < page_size; i++) { + if (bytes[i] != expected) { + platform_error_log("cache_test: unexpected byte at addr=%lu " + "offset=%lu: got=%u expected=%u\n", + addr, + i, + bytes[i], + expected); + rc = STATUS_TEST_FAILED; + break; + } + } + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +/* + * Corrupt one full raw-I/O page and make that corruption durable. The + * allocator clean-state records are intentionally outside the cache, so this + * lets the bootstrap test exercise A/B fallback without test-only allocator + * hooks. + */ +static platform_status +cache_test_zero_disk_page(io_handle *io, uint64 addr, uint64 page_size) +{ + buffer_handle buffer; + platform_status rc = platform_buffer_init(&buffer, page_size); + if (!SUCCESS(rc)) { + return rc; + } + + void *page = platform_buffer_getaddr(&buffer); + memset(page, 0, page_size); + rc = io_write(io, page, page_size, addr); + if (SUCCESS(rc)) { + rc = io_durable_barrier(io); + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +/* + * The recovery allocator must bootstrap only the extents it owns itself, then + * let recovery walkers rebuild all other ownership. In particular, it must + * not trust a refcount table persisted by an earlier clean shutdown. + */ +static platform_status +test_rc_allocator_recovery_bootstrap(allocator_config *cfg, + io_handle *io, + platform_heap_id hid) +{ + const allocator_root_id root_id = 1; + uint64 refcount_buffer_size = + ROUNDUP(cfg->extent_capacity * sizeof(refcount), cfg->io_cfg->page_size); + uint64 refcount_extent_count = + (refcount_buffer_size + cfg->io_cfg->extent_size - 1) + / cfg->io_cfg->extent_size; + uint64 reserved_extent_count = 1 + refcount_extent_count + 2; + uint64 old_clean_state_addr = + (1 + refcount_extent_count + 1) * cfg->io_cfg->extent_size; + uint64 super_addr = 0; + uint64 stale_extent_addr = 0; + platform_status rc = STATUS_OK; + rc_allocator original, recovery, remounted; + bool32 original_live = FALSE; + bool32 recovery_live = FALSE; + bool32 remounted_live = FALSE; + + ZERO_CONTENTS(&original); + ZERO_CONTENTS(&recovery); + ZERO_CONTENTS(&remounted); + platform_default_log("cache_test: allocator recovery bootstrap test started\n"); + + rc = rc_allocator_init( + &original, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + original_live = TRUE; + + rc = allocator_alloc_super_addr( + (allocator *)&original, root_id, &super_addr); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = allocator_alloc( + (allocator *)&original, &stale_extent_addr, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* Persist a non-reserved extent that the recovery mount must ignore. */ + rc_allocator_unmount(&original); + original_live = FALSE; + + /* + * New allocator initialization writes false records with sequences 1 and + * 2. The first clean close writes the true record into slot 0 (sequence + * 3), making slot 1 the older false record. Destroying that older slot + * must not prevent a normal mount from trusting the surviving true slot. + */ + rc = cache_test_zero_disk_page( + io, old_clean_state_addr, cfg->io_cfg->page_size); + if (!SUCCESS(rc)) { + goto cleanup; + } + + rc = rc_allocator_mount( + &remounted, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + platform_error_log("cache_test: normal mount did not fall back to its " + "surviving clean-state record\n"); + goto cleanup; + } + remounted_live = TRUE; + if (allocator_get_refcount((allocator *)&remounted, stale_extent_addr) + != AL_ONE_REF) + { + platform_error_log("cache_test: normal mount did not load the clean " + "refcount map\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + /* Simulate a crash after normal mount durably marked the allocator dirty. */ + rc_allocator_deinit(&remounted); + remounted_live = FALSE; + + rc = rc_allocator_mount( + &remounted, cfg, io, hid, platform_get_module_id()); + if (rc.r != STATUS_INVALID_STATE.r) { + platform_error_log("cache_test: normal mount trusted an allocator after " + "a simulated mounted crash\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = STATUS_OK; + + rc = rc_allocator_mount_recovery( + &recovery, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_live = TRUE; + + uint64 mounted_super_addr = 0; + rc = allocator_get_super_addr( + (allocator *)&recovery, root_id, &mounted_super_addr); + if (!SUCCESS(rc) || mounted_super_addr != super_addr) { + platform_error_log("cache_test: recovery lost a persisted superblock " + "mapping\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + if (allocator_in_use((allocator *)&recovery) != reserved_extent_count) { + platform_error_log("cache_test: recovery expected %lu reserved extents, " + "found %lu\n", + reserved_extent_count, + allocator_in_use((allocator *)&recovery)); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) + { + uint64 extent_addr = extent_no * cfg->io_cfg->extent_size; + if (allocator_get_refcount((allocator *)&recovery, extent_addr) + != AL_ONE_REF) + { + platform_error_log("cache_test: recovery did not reserve extent %lu\n", + extent_no); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + } + if (allocator_get_refcount((allocator *)&recovery, stale_extent_addr) + != AL_FREE) + { + platform_error_log("cache_test: recovery reused a persisted data " + "refcount\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + if (!SUCCESS(rc) + || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) + != AL_ONE_REF) + { + platform_error_log("cache_test: first rebuilt extent reference was " + "incorrect\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + /* + * Abort never persists a partial rebuild and leaves the allocator marked + * unclean, so a normal mount must refuse to trust the old refcount map. + */ + rc_allocator_abort_recovery(&recovery); + recovery_live = FALSE; + + rc = rc_allocator_mount( + &remounted, cfg, io, hid, platform_get_module_id()); + if (rc.r != STATUS_INVALID_STATE.r) { + platform_error_log("cache_test: normal mount trusted an unclean " + "allocator map\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = STATUS_OK; + + rc = rc_allocator_mount_recovery( + &recovery, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_live = TRUE; + + rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + if (!SUCCESS(rc) + || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) + != AL_ONE_REF + 1) + { + platform_error_log("cache_test: rebuilt extent reference did not " + "increment\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc_allocator_rebuild_finish(&recovery); + rc_allocator_unmount(&recovery); + recovery_live = FALSE; + + rc = rc_allocator_mount( + &remounted, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + remounted_live = TRUE; + if (allocator_get_refcount((allocator *)&remounted, stale_extent_addr) + != AL_ONE_REF + 1) + { + platform_error_log("cache_test: finished recovery was not persisted by " + "clean unmount\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + +cleanup: + if (remounted_live) { + rc_allocator_deinit(&remounted); + } + if (recovery_live) { + if (recovery.recovery_in_progress) { + rc_allocator_abort_recovery(&recovery); + } else { + rc_allocator_deinit(&recovery); + } + } + if (original_live) { + rc_allocator_deinit(&original); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: allocator recovery bootstrap test " + "passed\n"); + } + return rc; +} + +typedef struct { + cache *cc; + volatile bool32 finished; + platform_status status; +} cache_fence_test_context; + +/* Mirrors clockcache's private CC_WRITEBACK_REQUESTED status bit. */ +#define CACHE_TEST_WRITEBACK_REQUESTED (1u << 7) + +static void +cache_test_writeback_fence_thread(void *arg) +{ + cache_fence_test_context *ctxt = (cache_fence_test_context *)arg; + ctxt->status = cache_writeback_fence(ctxt->cc); + __atomic_store_n(&ctxt->finished, TRUE, __ATOMIC_RELEASE); +} + +/* + * Verify that a fence drains an already-dirty page without waiting for a page + * dirtied after its cut. Keeping old_page write-locked forces the fence to + * remain in progress long enough to deterministically dirty new_page after + * the dirty-generation rotation. + */ +static platform_status +test_cache_writeback_fence(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 pages_per_extent = + cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *old_page = NULL; + page_handle *new_page = NULL; + bool32 old_claimed = FALSE, old_locked = FALSE; + bool32 new_claimed = FALSE, new_locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 old_baseline = 0x11, old_value = 0x22; + const uint8 new_baseline = 0x33, new_value = 0x44; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread fence_thread; + + platform_default_log("cache_test: writeback fence test started\n"); + platform_assert(cfg->page_capacity >= 2 * pages_per_extent); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, 2 * pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 2); + if (!SUCCESS(rc)) { + platform_free(hid, addr_arr); + addr_arr = NULL; + goto cleanup; + } + + rc = cache_test_fill_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), old_baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_fill_page(cc, + addr_arr[pages_per_extent], + cache_config_page_size(&cfg->super), + new_baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + old_page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, old_page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + old_claimed = TRUE; + cache_lock(cc, old_page); + old_locked = TRUE; + memset(old_page->data, + old_value, + cache_config_page_size(&cfg->super)); + cache_mark_dirty(cc, old_page); + + clockcache *clock = (clockcache *)cc; + uint64 initial_generation = + __atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE); + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_thread_started = TRUE; + + timestamp wait_start = platform_get_timestamp(); + while (__atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE) + == initial_generation) + { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { + platform_error_log("cache_test: writeback fence did not take a cut\n"); + rc = STATUS_TIMEDOUT; + goto cleanup; + } + platform_sleep_ns(1000); + } + + new_page = cache_get( + cc, addr_arr[pages_per_extent], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, new_page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + new_claimed = TRUE; + cache_lock(cc, new_page); + new_locked = TRUE; + memset(new_page->data, + new_value, + cache_config_page_size(&cfg->super)); + cache_mark_dirty(cc, new_page); + cache_unlock(cc, new_page); + new_locked = FALSE; + cache_unclaim(cc, new_page); + new_claimed = FALSE; + cache_unget(cc, new_page); + new_page = NULL; + + if (__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + platform_error_log("cache_test: fence completed while old page was locked\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + +cleanup: + if (new_locked) { + cache_unlock(cc, new_page); + } + if (new_claimed) { + cache_unclaim(cc, new_page); + } + if (new_page != NULL) { + cache_unget(cc, new_page); + } + if (old_locked) { + cache_unlock(cc, old_page); + } + if (old_claimed) { + cache_unclaim(cc, old_page); + } + if (old_page != NULL) { + cache_unget(cc, old_page); + } + if (fence_thread_started) { + platform_status join_rc = platform_thread_join(&fence_thread); + if (!SUCCESS(join_rc) && SUCCESS(rc)) { + rc = join_rc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + if (SUCCESS(rc)) { + uint32 dirty_count = cache_count_dirty(cc); + if (dirty_count != 1) { + platform_error_log("cache_test: expected one post-cut dirty page, " + "found %u\n", + dirty_count); + rc = STATUS_TEST_FAILED; + } + } + if (SUCCESS(rc)) { + rc = cache_durable_barrier(cc); + } + if (SUCCESS(rc)) { + rc = cache_test_verify_disk_page(cc, + addr_arr[0], + cache_config_page_size(&cfg->super), + old_value); + } + if (SUCCESS(rc)) { + /* The post-cut page remains dirty, so its baseline must still be disk. */ + rc = cache_test_verify_disk_page(cc, + addr_arr[pages_per_extent], + cache_config_page_size(&cfg->super), + new_baseline); + } + + if (addr_arr != NULL) { + cache_flush(cc); + for (uint32 i = 0; i < 2; i++) { + uint64 addr = addr_arr[i * pages_per_extent]; + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr, PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + } + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: writeback fence test passed\n"); + } else { + platform_default_log("cache_test: writeback fence test failed\n"); + } + return rc; +} + +/* + * A fence must allow an already-claimed writer to finish its selected dirty + * interval, but it must not complete before that writer releases the claim. + */ +static platform_status +test_cache_writeback_fence_claimed_page(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 pages_per_extent = + cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *page = NULL; + bool32 claimed = FALSE, locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 baseline = 0x55, final_value = 0x66; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread fence_thread; + + platform_default_log("cache_test: claimed-page fence test started\n"); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 1); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_fill_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + claimed = TRUE; + cache_lock(cc, page); + locked = TRUE; + memset(page->data, final_value, cache_config_page_size(&cfg->super)); + cache_unlock(cc, page); + locked = FALSE; + + clockcache *clock = (clockcache *)cc; + uint64 initial_generation = + __atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE); + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_thread_started = TRUE; + + uint64 entry_no = + clock->lookup[addr_arr[0] >> clock->cfg->log_page_size]; + platform_assert(entry_no < clock->cfg->page_capacity); + timestamp wait_start = platform_get_timestamp(); + while ((__atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE) + == initial_generation) + || !(__atomic_load_n(&clock->entry[entry_no].status, + __ATOMIC_ACQUIRE) + & CACHE_TEST_WRITEBACK_REQUESTED)) + { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { + platform_error_log("cache_test: fence did not request claimed page\n"); + rc = STATUS_TIMEDOUT; + goto cleanup; + } + platform_sleep_ns(1000); + } + + if (__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + platform_error_log("cache_test: fence completed while page was claimed\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + /* This claim predates the request, so it is permitted to finish. */ + cache_lock(cc, page); + locked = TRUE; + memset(page->data, final_value, cache_config_page_size(&cfg->super)); + cache_unlock(cc, page); + locked = FALSE; + cache_unclaim(cc, page); + claimed = FALSE; + cache_unget(cc, page); + page = NULL; + +cleanup: + if (locked) { + cache_unlock(cc, page); + } + if (claimed) { + cache_unclaim(cc, page); + } + if (page != NULL) { + cache_unget(cc, page); + } + if (fence_thread_started) { + platform_status join_rc = platform_thread_join(&fence_thread); + if (!SUCCESS(join_rc) && SUCCESS(rc)) { + rc = join_rc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + if (SUCCESS(rc) && cache_count_dirty(cc) != 0) { + platform_error_log("cache_test: claimed-page fence left dirty pages\n"); + rc = STATUS_TEST_FAILED; + } + if (SUCCESS(rc)) { + rc = cache_durable_barrier(cc); + } + if (SUCCESS(rc)) { + rc = cache_test_verify_disk_page(cc, + addr_arr[0], + cache_config_page_size(&cfg->super), + final_value); + } + + if (addr_arr != NULL) { + cache_flush(cc); + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr_arr[0], PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: claimed-page fence test passed\n"); + } else { + platform_default_log("cache_test: claimed-page fence test failed\n"); + } + return rc; +} + platform_status test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) { @@ -98,6 +780,16 @@ test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) page_handle **page_arr = NULL; uint64 *addr_arr = NULL; + rc = test_cache_writeback_fence(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + + rc = test_cache_writeback_fence_claimed_page(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + /* allocate twice as many pages as the cache capacity */ uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); uint32 extent_capacity = cfg->page_capacity / pages_per_extent; @@ -977,6 +1669,12 @@ cache_test(int argc, char *argv[]) goto cleanup; } + rc = test_rc_allocator_recovery_bootstrap( + &system_cfg.allocator_cfg, io, hid); + if (!SUCCESS(rc)) { + goto destroy_iohandle; + } + rc = test_init_task_system(&ts, hid, &system_cfg.task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n", diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index 1bf96ba52..6a16d05f5 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -19,6 +19,8 @@ #include "poison.h" +#define LOG_TEST_LEAVES_PER_MEMTABLE 97 + int test_log_crash(clockcache *cc, clockcache_config *cache_cfg, @@ -59,12 +61,45 @@ test_log_crash(clockcache *cc, merge_accumulator_init(&msg, hid); for (i = 0; i < num_entries; i++) { + uint64 entry_num = i; + if (2 * LOG_TEST_LEAVES_PER_MEMTABLE <= num_entries + && i < 2 * LOG_TEST_LEAVES_PER_MEMTABLE) + { + /* + * Write the first two generations in the opposite order, and each + * generation in reverse leaf order. The iterator must restore the + * (memtable_generation, leaf_generation) order below. + */ + uint64 memtable_generation = 1 - i / LOG_TEST_LEAVES_PER_MEMTABLE; + uint64 leaf_generation = LOG_TEST_LEAVES_PER_MEMTABLE - 1 + - i % LOG_TEST_LEAVES_PER_MEMTABLE; + entry_num = memtable_generation * LOG_TEST_LEAVES_PER_MEMTABLE + + leaf_generation; + } key skey = - test_key(&keybuffer, TEST_RANDOM, i, 0, 0, 1 + (i % key_size), 0); - generate_test_message(gen, i, &msg); - log_write(logh, skey, merge_accumulator_to_message(&msg), i); + test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + int log_rc = log_write(logh, + skey, + merge_accumulator_to_message(&msg), + entry_num / LOG_TEST_LEAVES_PER_MEMTABLE, + entry_num % LOG_TEST_LEAVES_PER_MEMTABLE); + platform_assert(log_rc == 0); } + rc = log_seal(logh); + platform_assert_status_ok(rc); + rc = cache_writeback_fence((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + if (crash) { clockcache_deinit(cc); rc = clockcache_init( @@ -82,6 +117,13 @@ test_log_crash(clockcache *cc, generate_test_message(gen, i, &msg); message mmessage = merge_accumulator_to_message(&msg); iterator_curr(itorh, &returned_key, &returned_message); + uint64 memtable_generation; + uint64 leaf_generation; + shard_log_iterator_curr_generations( + &itor, &memtable_generation, &leaf_generation); + platform_assert( + memtable_generation == i / LOG_TEST_LEAVES_PER_MEMTABLE); + platform_assert(leaf_generation == i % LOG_TEST_LEAVES_PER_MEMTABLE); if (data_key_compare(cfg->data_cfg, skey, returned_key) || message_lex_cmp(mmessage, returned_message)) { @@ -99,6 +141,8 @@ test_log_crash(clockcache *cc, } platform_default_log("log returned %lu of %lu entries\n", i, num_entries); + platform_assert(i == num_entries); + platform_assert(!iterator_can_curr(itorh)); merge_accumulator_deinit(&msg); @@ -108,6 +152,182 @@ test_log_crash(clockcache *cc, return 0; } +static void +test_log_write_range(log_handle *logh, + test_message_generator *gen, + platform_heap_id hid, + uint64 key_size, + uint64 first_entry, + uint64 num_entries) +{ + merge_accumulator msg; + DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); + merge_accumulator_init(&msg, hid); + + for (uint64 i = 0; i < num_entries; i++) { + uint64 entry_num = first_entry + i; + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + int log_rc = log_write(logh, + skey, + merge_accumulator_to_message(&msg), + entry_num, + 0); + platform_assert(log_rc == 0); + } + + merge_accumulator_deinit(&msg); +} + +static void +test_log_verify_segment(cache *cc, + shard_log_config *cfg, + const log_segment_info *segment, + test_message_generator *gen, + platform_heap_id hid, + uint64 key_size, + uint64 first_entry, + uint64 num_entries) +{ + shard_log_iterator itor; + iterator *itorh = (iterator *)&itor; + merge_accumulator msg; + DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); + key returned_key; + message returned_message; + + platform_assert(segment->addr != 0); + platform_assert(segment->meta_addr != 0); + platform_status rc = shard_log_iterator_init( + cc, cfg, hid, segment->addr, segment->magic, &itor); + platform_assert_status_ok(rc); + + merge_accumulator_init(&msg, hid); + for (uint64 i = 0; i < num_entries; i++) { + uint64 entry_num = first_entry + i; + platform_assert(iterator_can_curr(itorh)); + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + iterator_curr(itorh, &returned_key, &returned_message); + uint64 memtable_generation; + uint64 leaf_generation; + shard_log_iterator_curr_generations( + &itor, &memtable_generation, &leaf_generation); + platform_assert(memtable_generation == entry_num); + platform_assert(leaf_generation == 0); + platform_assert(data_key_compare(cfg->data_cfg, skey, returned_key) == 0); + platform_assert(message_lex_cmp( + merge_accumulator_to_message(&msg), returned_message) + == 0); + rc = iterator_next(itorh); + platform_assert_status_ok(rc); + } + platform_assert(!iterator_can_curr(itorh)); + + merge_accumulator_deinit(&msg); + shard_log_iterator_deinit(hid, &itor); +} + +/* + * A rotation must permanently detach the first mini-allocator stream and + * produce a fresh one. Reinitializing the cache after each forced physical + * persistence cut makes this test exercise only persisted pages for both + * identities; it does not model logical durable-tail publication. + */ +static int +test_log_rotate(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + shard_log *log, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + const uint64 old_first = 1000, old_count = 16; + const uint64 new_first = 2000, new_count = 16; + log_segment_info sealed, fresh; + + platform_status rc = shard_log_init(log, (cache *)cc, cfg); + platform_assert_status_ok(rc); + + test_log_write_range( + (log_handle *)log, gen, hid, key_size, old_first, old_count); + rc = log_rotate((log_handle *)log, &sealed, &fresh); + platform_assert_status_ok(rc); + platform_assert(sealed.addr != 0); + platform_assert(sealed.meta_addr != 0); + platform_assert(fresh.addr != 0); + platform_assert(fresh.meta_addr != 0); + platform_assert(sealed.meta_addr != fresh.meta_addr); + platform_assert(sealed.magic != fresh.magic); + + rc = cache_writeback_fence((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + + clockcache_deinit(cc); + rc = clockcache_init( + cc, cache_cfg, io, al, "rotated-old", hid, platform_get_module_id()); + platform_assert_status_ok(rc); + test_log_verify_segment((cache *)cc, + cfg, + &sealed, + gen, + hid, + key_size, + old_first, + old_count); + + test_log_write_range( + (log_handle *)log, gen, hid, key_size, new_first, new_count); + rc = log_seal((log_handle *)log); + platform_assert_status_ok(rc); + rc = cache_writeback_fence((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + + clockcache_deinit(cc); + rc = clockcache_init( + cc, cache_cfg, io, al, "rotated-new", hid, platform_get_module_id()); + platform_assert_status_ok(rc); + test_log_verify_segment((cache *)cc, + cfg, + &sealed, + gen, + hid, + key_size, + old_first, + old_count); + test_log_verify_segment((cache *)cc, + cfg, + &fresh, + gen, + hid, + key_size, + new_first, + new_count); + + shard_log_segment_discard((cache *)cc, &sealed); + shard_log_zap(log); + return 0; +} + static int test_log_large_message(cache *cc, shard_log_config *cfg, @@ -134,7 +354,8 @@ test_log_large_message(cache *cc, memset(merge_accumulator_data(&msg), 'L', value_len); int log_rc = - log_write((log_handle *)log, skey, merge_accumulator_to_message(&msg), 0); + log_write( + (log_handle *)log, skey, merge_accumulator_to_message(&msg), 0, 0); platform_assert(log_rc == 0); merge_accumulator filler; @@ -146,11 +367,22 @@ test_log_large_message(cache *cc, merge_accumulator_data(&filler), 'f', merge_accumulator_length(&filler)); for (uint64 i = 1; i < 16; i++) { log_rc = log_write( - (log_handle *)log, skey, merge_accumulator_to_message(&filler), i); + (log_handle *)log, + skey, + merge_accumulator_to_message(&filler), + i / 4, + i % 4); platform_assert(log_rc == 0); } merge_accumulator_deinit(&filler); + rc = log_seal((log_handle *)log); + platform_assert_status_ok(rc); + rc = cache_writeback_fence(cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier(cc); + platform_assert_status_ok(rc); + rc = shard_log_iterator_init(cc, cfg, hid, @@ -201,7 +433,12 @@ test_log_thread(void *arg) for (i = thread_id * num_entries; i < (thread_id + 1) * num_entries; i++) { key skey = test_key(&keybuf, TEST_RANDOM, i, 0, 0, key_size, 0); generate_test_message(gen, i, &msg); - log_write(logh, skey, merge_accumulator_to_message(&msg), i); + int log_rc = log_write(logh, + skey, + merge_accumulator_to_message(&msg), + i / 1024, + i % 1024); + platform_assert(log_rc == 0); } merge_accumulator_deinit(&msg); @@ -380,6 +617,17 @@ log_test(int argc, char *argv[]) rc = test_log_large_message((cache *)cc, &system_cfg.log_cfg, log, hid); platform_assert(rc == 0); + rc = test_log_rotate(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + log, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); + if (run_perf_test) { ret = test_log_perf((cache *)cc, &system_cfg.log_cfg, diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 5377cde9f..9c550ec09 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -250,6 +250,85 @@ CTEST2(splinter, test_inserts) core_destroy(&spl); } +/* + * The second checkpoint slot is a torn-write fallback, not permission for a + * normal mount to silently roll back past a newer, valid active record. A + * successful mount publishes such an active record; until crash recovery is + * implemented, a concurrent/restarted normal mount must reject it even + * though the preceding clean record remains valid in the other slot. + */ +CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) +{ + allocator *alp = (allocator *)&data->al; + allocator_root_id root_id = test_generate_allocator_root_id(); + core_handle created, mounted, rejected, cleanup; + platform_status rc; + + rc = core_mkfs(&created, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + /* Give the clean record a real COW root, not just the empty-tree root. */ + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + test_key(&keybuf, + TEST_RANDOM, + 1, + 0, + 0, + data->workload_cfg->key_size, + 0); + generate_test_message(&data->gen, 1, &msg); + rc = core_insert(&created, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + merge_accumulator_deinit(&msg); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_unmount(&created); + ASSERT_TRUE(SUCCESS(rc)); + + /* This mount advances the A/B sequence with an unmounted=FALSE record. */ + rc = core_mount(&mounted, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_mount(&rejected, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); + + /* Finish cleanly, then prove the same record pair is mountable again. */ + rc = core_unmount(&mounted); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_mount(&cleanup, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + core_destroy(&cleanup); +} + static void trunk_shadow_init(trunk_shadow *shadow, data_config *data_cfg, From 4c6c6829432ac8192acd575c85ae64b2280bdf1c Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 13:22:25 -0700 Subject: [PATCH 02/13] clarify durability semantics Signed-off-by: Rob Johnson --- src/blob.c | 2 +- src/cache.h | 55 ++++++++++++++----------- src/clockcache.c | 69 ++++++++++++++++++-------------- src/core.c | 4 +- src/platform_linux/platform_io.h | 8 +++- src/shard_log.c | 2 +- 6 files changed, 82 insertions(+), 58 deletions(-) diff --git a/src/blob.c b/src/blob.c index b6c8e084b..9766c4dc5 100644 --- a/src/blob.c +++ b/src/blob.c @@ -327,7 +327,7 @@ blob_sync(cache *cc, slice sblob) if (!SUCCESS(rc)) { break; } - cache_page_sync(cc, itor.page, FALSE, PAGE_TYPE_BLOB); + cache_page_writeback(cc, itor.page, FALSE, PAGE_TYPE_BLOB); blob_page_iterator_advance_page(&itor); } diff --git a/src/cache.h b/src/cache.h index 52bec1434..a5c27e686 100644 --- a/src/cache.h +++ b/src/cache.h @@ -130,13 +130,13 @@ typedef async_status (*page_get_async_fn)(void *payload); typedef page_handle *(*page_get_async_state_result_fn)(void *payload); typedef bool32 (*page_try_claim_fn)(cache *cc, page_handle *page); -typedef void (*page_sync_fn)(cache *cc, - page_handle *page, - bool32 is_blocking, - page_type type); -typedef void (*extent_sync_fn)(cache *cc, - uint64 addr, - uint64 *pages_outstanding); +typedef void (*page_writeback_fn)(cache *cc, + page_handle *page, + bool32 is_blocking, + page_type type); +typedef void (*extent_writeback_fn)(cache *cc, + uint64 addr, + uint64 *pages_outstanding); typedef void (*page_prefetch_fn)(cache *cc, uint64 addr, page_type type); typedef int (*evict_fn)(cache *cc, bool32 ignore_pinned); typedef bool32 (*page_addr_pred_fn)(cache *cc, uint64 addr); @@ -175,8 +175,8 @@ typedef struct cache_ops { page_generic_fn page_mark_dirty; page_generic_fn page_pin; page_generic_fn page_unpin; - page_sync_fn page_sync; - extent_sync_fn extent_sync; + page_writeback_fn page_writeback; + extent_writeback_fn extent_writeback; cache_generic_fn flush; cache_writeback_fence_fn writeback_fence; cache_durable_barrier_fn durable_barrier; @@ -489,37 +489,44 @@ cache_unpin(cache *cc, page_handle *page) /* *----------------------------------------------------------------------------- - * cache_page_sync + * cache_page_writeback * - * Asynchronously writes the page back to disk. + * Issues writeback of the page to disk. This does NOT make the page durable; + * it only hands the write to the I/O layer (no device-cache flush). * - * This is used to sync log pages opportunistically. The current API doesn't - * inform the user when this happens. "It's not the ideal API." -- @aconway + * With is_blocking == FALSE the writeback is issued asynchronously and this + * returns without waiting for completion; there is no per-call way to observe + * when it finishes. With is_blocking == TRUE the page is written synchronously + * and the write has completed by the time this returns. *----------------------------------------------------------------------------- */ static inline void -cache_page_sync(cache *cc, - page_handle *page, - bool32 is_blocking, - page_type type) +cache_page_writeback(cache *cc, + page_handle *page, + bool32 is_blocking, + page_type type) { - return cc->ops->page_sync(cc, page, is_blocking, type); + return cc->ops->page_writeback(cc, page, is_blocking, type); } /* *----------------------------------------------------------------------------- - * cache_extent_sync + * cache_extent_writeback * - * Asynchronously syncs the extent beginning at addr. + * Asynchronously issues writeback of the extent beginning at addr. This does + * NOT make the extent durable; it only hands the writes to the I/O layer (no + * device-cache flush) and returns without waiting for completion. * * *pages_outstanding is immediately incremented by the number of pages * issued for writeback (the non-clean pages of the extent); as writebacks - * complete, *pages_outstanding is decremented atomically. + * complete, *pages_outstanding is decremented atomically. This counter is the + * only way to observe when the issued writebacks finish. * * Assumes pages_outstanding is an aligned uint64, so (on x86) the caller * can access it and observe its value atomically. * - * TODO: What happens if two callers call cache_extent_sync on the same extent? + * TODO: What happens if two callers call cache_extent_writeback on the same + * extent? * * All pages in the extent must be clean or cleanable. * The page may not be in writeback, loading, or locked, or claimed, otherwise @@ -527,9 +534,9 @@ cache_page_sync(cache *cc, *----------------------------------------------------------------------------- */ static inline void -cache_extent_sync(cache *cc, uint64 addr, uint64 *pages_outstanding) +cache_extent_writeback(cache *cc, uint64 addr, uint64 *pages_outstanding) { - cc->ops->extent_sync(cc, addr, pages_outstanding); + cc->ops->extent_writeback(cc, addr, pages_outstanding); } /* diff --git a/src/clockcache.c b/src/clockcache.c index 74aba087b..760a2a708 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -2540,17 +2540,22 @@ clockcache_unpin(clockcache *cc, page_handle *page) /* *----------------------------------------------------------------------------- - * clockcache_page_sync -- + * clockcache_page_writeback -- * - * Asynchronously syncs the page. Currently there is no way to check - *when the writeback has completed. + * Issues writeback of the page. This does not make the page durable; it + * only hands the write to the I/O layer. + * + * With is_blocking == FALSE the writeback is issued asynchronously and + * this returns without waiting for completion. With is_blocking == TRUE + * the page is written synchronously and has completed by the time this + * returns. *----------------------------------------------------------------------------- */ void -clockcache_page_sync(clockcache *cc, - page_handle *page, - bool32 is_blocking, - page_type type) +clockcache_page_writeback(clockcache *cc, + page_handle *page, + bool32 is_blocking, + page_type type) { uint32 entry_number = clockcache_page_to_entry_number(cc, page); async_io_state *state; @@ -2574,7 +2579,7 @@ clockcache_page_sync(clockcache *cc, } platform_assert(0, - "page_sync requires a cleanable page: entry=%u " + "page_writeback requires a cleanable page: entry=%u " "status=%u\n", entry_number, clockcache_get_status(cc, entry_number)); @@ -2588,7 +2593,7 @@ clockcache_page_sync(clockcache *cc, if (!is_blocking) { state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); if (state == NULL) { - platform_error_log("clockcache_page_sync: async_io_state allocation " + platform_error_log("clockcache_page_writeback: async_io_state allocation " "failed for addr %lu, entry %u, type %u\n", addr, entry_number, @@ -2604,7 +2609,7 @@ clockcache_page_sync(clockcache *cc, clockcache_write_callback, state); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_async_state_init failed " + platform_error_log("clockcache_page_writeback: io_async_state_init failed " "for addr %lu, entry %u, type %u: %s\n", addr, entry_number, @@ -2614,7 +2619,7 @@ clockcache_page_sync(clockcache *cc, platform_assert_status_ok(status); status = io_async_state_append_page(state->iostate, page->data); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_async_state_append_page " + platform_error_log("clockcache_page_writeback: io_async_state_append_page " "failed for addr %lu, entry %u, type %u: %s\n", addr, entry_number, @@ -2626,7 +2631,7 @@ clockcache_page_sync(clockcache *cc, } else { status = io_write(cc->io, page->data, clockcache_page_size(cc), addr); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_write failed for addr " + platform_error_log("clockcache_page_writeback: io_write failed for addr " "%lu, entry %u, type %u: %s\n", addr, entry_number, @@ -2636,7 +2641,7 @@ clockcache_page_sync(clockcache *cc, platform_assert_status_ok(status); clockcache_log(addr, entry_number, - "page_sync write entry %u addr %lu\n", + "page_writeback write entry %u addr %lu\n", entry_number, addr); clockcache_dirty_complete_writeback(cc, entry_number); @@ -2645,20 +2650,24 @@ clockcache_page_sync(clockcache *cc, /* *----------------------------------------------------------------------------- - * clockcache_extent_sync -- + * clockcache_extent_writeback -- * - * Asynchronously syncs the extent. + * Asynchronously issues writeback of the extent. This does not make the + * extent durable; it only hands the writes to the I/O layer and returns + * without waiting for completion. * * Adds the number of pages issued writeback to the counter pointed to * by pages_outstanding. When the writes complete, a callback subtracts * them off, so that the caller may track how many pages are in - *writeback. + * writeback. * * Assumes all pages in the extent are clean or cleanable *----------------------------------------------------------------------------- */ void -clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) +clockcache_extent_writeback(clockcache *cc, + uint64 addr, + uint64 *pages_outstanding) { async_io_state *state = NULL; uint64 i; @@ -2677,7 +2686,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) req_addr = page_addr; state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); if (state == NULL) { - platform_error_log("clockcache_extent_sync: async_io_state " + platform_error_log("clockcache_extent_writeback: async_io_state " "allocation failed for extent addr %lu, " "page addr %lu, entry %u\n", addr, @@ -2694,7 +2703,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) clockcache_write_callback, state); if (!SUCCESS(rc)) { - platform_error_log("clockcache_extent_sync: " + platform_error_log("clockcache_extent_writeback: " "io_async_state_init failed for extent addr " "%lu, req addr %lu, entry %u: %s\n", addr, @@ -2707,7 +2716,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) platform_status rc = io_async_state_append_page( state->iostate, clockcache_get_entry(cc, entry_number)->page.data); if (!SUCCESS(rc)) { - platform_error_log("clockcache_extent_sync: " + platform_error_log("clockcache_extent_writeback: " "io_async_state_append_page failed for extent " "addr %lu, page addr %lu, entry %u: %s\n", addr, @@ -3453,20 +3462,22 @@ clockcache_get_async_state_result_virtual(void *payload) } void -clockcache_page_sync_virtual(cache *c, - page_handle *page, - bool32 is_blocking, - page_type type) +clockcache_page_writeback_virtual(cache *c, + page_handle *page, + bool32 is_blocking, + page_type type) { clockcache *cc = (clockcache *)c; - clockcache_page_sync(cc, page, is_blocking, type); + clockcache_page_writeback(cc, page, is_blocking, type); } void -clockcache_extent_sync_virtual(cache *c, uint64 addr, uint64 *pages_outstanding) +clockcache_extent_writeback_virtual(cache *c, + uint64 addr, + uint64 *pages_outstanding) { clockcache *cc = (clockcache *)c; - clockcache_extent_sync(cc, addr, pages_outstanding); + clockcache_extent_writeback(cc, addr, pages_outstanding); } void @@ -3621,8 +3632,8 @@ static cache_ops clockcache_ops = { .page_mark_dirty = clockcache_mark_dirty_virtual, .page_pin = clockcache_pin_virtual, .page_unpin = clockcache_unpin_virtual, - .page_sync = clockcache_page_sync_virtual, - .extent_sync = clockcache_extent_sync_virtual, + .page_writeback = clockcache_page_writeback_virtual, + .extent_writeback = clockcache_extent_writeback_virtual, .flush = clockcache_flush_virtual, .writeback_fence = clockcache_writeback_fence_virtual, .durable_barrier = clockcache_durable_barrier_virtual, diff --git a/src/core.c b/src/core.c index 3faac28ed..c86587994 100644 --- a/src/core.c +++ b/src/core.c @@ -277,7 +277,7 @@ core_write_checkpoint_page(core_handle *spl, cache_mark_dirty(spl->cc, page); cache_unlock(spl->cc, page); cache_unclaim(spl->cc, page); - cache_page_sync(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); + cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); cache_unget(spl->cc, page); } @@ -290,7 +290,7 @@ core_initialize_checkpoint_record_page(core_handle *spl, uint64 page_addr) cache_mark_dirty(spl->cc, page); cache_unlock(spl->cc, page); cache_unclaim(spl->cc, page); - cache_page_sync(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); + cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); cache_unget(spl->cc, page); } diff --git a/src/platform_linux/platform_io.h b/src/platform_linux/platform_io.h index 3ecd12e09..cb60e861d 100644 --- a/src/platform_linux/platform_io.h +++ b/src/platform_linux/platform_io.h @@ -199,7 +199,13 @@ io_cleanup(io_handle *io, uint64 count) return io->ops->cleanup(io, count); } -// Guarantees all in-flight IOs are complete before return +// Wait until we have seen each process's I/O be quiescent. +// +// This means that all I/Os that were in-flight at the start of this call are +// guaranteed to have finished before this call returns. +// +// But I/Os that are issued after this call starts may not be complete when +// this call returns. static inline void io_wait_all(io_handle *io) { diff --git a/src/shard_log.c b/src/shard_log.c index 9c1fd0162..cccf8afe6 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -344,7 +344,7 @@ shard_log_write(log_handle *logh, cache_unlock(cc, page); cache_unclaim(cc, page); - cache_page_sync(cc, page, FALSE, PAGE_TYPE_LOG); + cache_page_writeback(cc, page, FALSE, PAGE_TYPE_LOG); cache_unget(cc, page); if (get_new_page_for_thread(log, thread_data, &page)) { From 6954ceb01375b46b7dc94e7204aef810cd2cb97d Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 13:51:50 -0700 Subject: [PATCH 03/13] get rid of mark_dirty Signed-off-by: Rob Johnson --- src/blob.c | 1 - src/btree.c | 3 -- src/cache.h | 92 +++++++++++++---------------------- src/clockcache.c | 29 ----------- src/core.c | 2 - src/mini_allocator.c | 1 - src/shard_log.c | 1 - src/trunk.c | 2 - tests/functional/cache_test.c | 4 -- 9 files changed, 34 insertions(+), 101 deletions(-) diff --git a/src/blob.c b/src/blob.c index 9766c4dc5..f8c0c375c 100644 --- a/src/blob.c +++ b/src/blob.c @@ -226,7 +226,6 @@ blob_page_iterator_get_curr(blob_page_iterator *iter, iter->cc, iter->fragment.addr, TRUE, PAGE_TYPE_BLOB); } cache_lock(iter->cc, iter->page); - cache_mark_dirty(iter->cc, iter->page); } } } diff --git a/src/btree.c b/src/btree.c index e93b0c50c..9504aba36 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1197,7 +1197,6 @@ btree_node_lock(cache *cc, // IN btree_node *node) // IN { cache_lock(cc, node->page); - cache_mark_dirty(cc, node->page); } static inline void @@ -1294,8 +1293,6 @@ btree_create(cache *cc, btree_init_hdr(cfg, root.hdr); - cache_mark_dirty(cc, root.page); - // If this btree is for a memtable then pin all pages belonging to it if (pinned) { cache_pin(cc, root.page); diff --git a/src/cache.h b/src/cache.h index a5c27e686..1cecd1318 100644 --- a/src/cache.h +++ b/src/cache.h @@ -165,37 +165,36 @@ typedef struct cache_ops { page_get_async_fn page_get_async; page_get_async_state_result_fn page_get_async_result; - page_generic_fn page_unget; - page_try_claim_fn page_try_claim; - page_generic_fn page_unclaim; - page_generic_fn page_lock; - page_generic_fn page_unlock; - page_prefetch_fn page_prefetch; - page_prefetch_fn page_prefetch_page; - page_generic_fn page_mark_dirty; - page_generic_fn page_pin; - page_generic_fn page_unpin; - page_writeback_fn page_writeback; - extent_writeback_fn extent_writeback; - cache_generic_fn flush; + page_generic_fn page_unget; + page_try_claim_fn page_try_claim; + page_generic_fn page_unclaim; + page_generic_fn page_lock; + page_generic_fn page_unlock; + page_prefetch_fn page_prefetch; + page_prefetch_fn page_prefetch_page; + page_generic_fn page_pin; + page_generic_fn page_unpin; + page_writeback_fn page_writeback; + extent_writeback_fn extent_writeback; + cache_generic_fn flush; cache_writeback_fence_fn writeback_fence; cache_durable_barrier_fn durable_barrier; - evict_fn evict; - cache_generic_fn cleanup; - page_addr_pred_fn in_use; - page_addr_fn assert_ungot; - cache_generic_fn assert_free; - validate_page_fn validate_page; - cache_present_fn cache_present; - cache_print_fn print; - cache_print_fn print_stats; - io_stats_fn io_stats; - cache_generic_fn reset_stats; - count_dirty_fn count_dirty; - page_get_read_ref_fn page_get_read_ref; - enable_sync_get_fn enable_sync_get; - get_allocator_fn get_allocator; - cache_config_fn get_config; + evict_fn evict; + cache_generic_fn cleanup; + page_addr_pred_fn in_use; + page_addr_fn assert_ungot; + cache_generic_fn assert_free; + validate_page_fn validate_page; + cache_present_fn cache_present; + cache_print_fn print; + cache_print_fn print_stats; + io_stats_fn io_stats; + cache_generic_fn reset_stats; + count_dirty_fn count_dirty; + page_get_read_ref_fn page_get_read_ref; + enable_sync_get_fn enable_sync_get; + get_allocator_fn get_allocator; + cache_config_fn get_config; } cache_ops; // To sub-class cache, make a cache your first field; @@ -370,10 +369,11 @@ cache_unclaim(cache *cc, page_handle *page) * * Blocks until outstanding read locks are released by other threads. * - * Clockcache conservatively begins a dirty interval on this transition, so a - * checkpoint fence cannot miss a caller that makes its first change before - * calling cache_mark_dirty(). cache_mark_dirty() remains the explicit, - * idempotent declaration of intent to modify the page. + * Acquiring the write lock marks the page dirty: it begins a dirty interval on + * this transition (before the caller's first change), so a checkpoint fence + * cannot miss the modification. A page obtained write-locked from cache_alloc() + * is likewise already dirty. Callers therefore do not separately declare the + * mutation. *---------------------------------------------------------------------- */ static inline void @@ -433,26 +433,6 @@ cache_prefetch_page(cache *cc, uint64 addr, page_type type) return cc->ops->page_prefetch_page(cc, addr, type); } -/* - *---------------------------------------------------------------------- - * cache_mark_dirty - * - * Marks a page changed, to be written back. - * The caller had better have the write lock on the page via cache_lock() - * before changing its value. - * - * Clockcache already begins a dirty interval when the caller acquires the - * write lock; this call is therefore idempotent there. It remains part of the - * cache API to document the mutation and preserve the contract for other - * implementations. - *---------------------------------------------------------------------- - */ -static inline void -cache_mark_dirty(cache *cc, page_handle *page) -{ - return cc->ops->page_mark_dirty(cc, page); -} - /* *---------------------------------------------------------------------- * cache_pin @@ -560,11 +540,7 @@ cache_flush(cache *cc) * * Wait until every page whose current dirty interval began before this call's * cut has completed writeback. Pages dirtied after the cut do not delay the - * call. The cache implementation must inspect resident metadata only; it must - * not fault pages in merely to satisfy the fence. - * - * This is the checkpointing primitive. It is intentionally narrower than - * cache_flush(), which attempts to leave the entire cache clean. + * call. *----------------------------------------------------------------------------- */ static inline platform_status diff --git a/src/clockcache.c b/src/clockcache.c index 760a2a708..44474e237 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -2477,27 +2477,6 @@ clockcache_unlock(clockcache *cc, page_handle *page) } -/*---------------------------------------------------------------------- - * clockcache_mark_dirty -- - * - * Marks the entry dirty. - *---------------------------------------------------------------------- - */ -void -clockcache_mark_dirty(clockcache *cc, page_handle *page) -{ - debug_only clockcache_entry *entry = clockcache_page_to_entry(cc, page); - uint32 entry_number = clockcache_page_to_entry_number(cc, page); - - clockcache_log(entry->page.disk_addr, - entry_number, - "mark_dirty: entry %u addr %lu\n", - entry_number, - entry->page.disk_addr); - clockcache_dirty_begin(cc, entry_number); - return; -} - /* *---------------------------------------------------------------------- * clockcache_pin -- @@ -3409,13 +3388,6 @@ clockcache_prefetch_page_virtual(cache *c, uint64 addr, page_type type) clockcache_prefetch_page(cc, addr, type); } -void -clockcache_mark_dirty_virtual(cache *c, page_handle *page) -{ - clockcache *cc = (clockcache *)c; - clockcache_mark_dirty(cc, page); -} - void clockcache_pin_virtual(cache *c, page_handle *page) { @@ -3629,7 +3601,6 @@ static cache_ops clockcache_ops = { .page_unlock = clockcache_unlock_virtual, .page_prefetch = clockcache_prefetch_virtual, .page_prefetch_page = clockcache_prefetch_page_virtual, - .page_mark_dirty = clockcache_mark_dirty_virtual, .page_pin = clockcache_pin_virtual, .page_unpin = clockcache_unpin_virtual, .page_writeback = clockcache_page_writeback_virtual, diff --git a/src/core.c b/src/core.c index c86587994..b91d6f2e7 100644 --- a/src/core.c +++ b/src/core.c @@ -274,7 +274,6 @@ core_write_checkpoint_page(core_handle *spl, platform_assert(contents_size <= cache_page_size(spl->cc)); memset(page->data, 0, cache_page_size(spl->cc)); memcpy(page->data, contents, contents_size); - cache_mark_dirty(spl->cc, page); cache_unlock(spl->cc, page); cache_unclaim(spl->cc, page); cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); @@ -287,7 +286,6 @@ core_initialize_checkpoint_record_page(core_handle *spl, uint64 page_addr) page_handle *page = cache_alloc(spl->cc, page_addr, PAGE_TYPE_SUPERBLOCK); platform_assert(page != NULL); memset(page->data, 0, cache_page_size(spl->cc)); - cache_mark_dirty(spl->cc, page); cache_unlock(spl->cc, page); cache_unclaim(spl->cc, page); cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); diff --git a/src/mini_allocator.c b/src/mini_allocator.c index fa350f786..3142a6a73 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -198,7 +198,6 @@ mini_full_lock_meta_tail(mini_allocator *mini) static void mini_full_unlock_meta_page(mini_allocator *mini, page_handle *meta_page) { - cache_mark_dirty(mini->cc, meta_page); cache_unlock(mini->cc, meta_page); cache_unclaim(mini->cc, meta_page); cache_unget(mini->cc, meta_page); diff --git a/src/shard_log.c b/src/shard_log.c index cccf8afe6..205ed3cad 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -434,7 +434,6 @@ shard_log_seal(log_handle *logh) log_entry_set_terminal(cursor); } hdr->checksum = shard_log_checksum(log->cfg, page); - cache_mark_dirty(cc, page); cache_unlock(cc, page); cache_unclaim(cc, page); diff --git a/src/trunk.c b/src/trunk.c index d24c17ae5..c95c0c72e 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -2008,7 +2008,6 @@ trunk_node_serialize_maybe_setup_next_page(cache *cc, "%s():%d: cache_alloc() failed", __func__, __LINE__); return STATUS_NO_MEMORY; } - cache_mark_dirty(cc, *current_page); *page_offset = 0; } @@ -2131,7 +2130,6 @@ trunk_node_serialize(trunk_context *context, trunk_node *node) rc = STATUS_NO_MEMORY; goto cleanup; } - cache_mark_dirty(context->cc, header_page); int64 min_inflight_bundle_start = trunk_node_first_live_inflight_bundle(node); diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index d75267d37..5eb6dd72f 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -489,7 +489,6 @@ test_cache_writeback_fence(cache *cc, memset(old_page->data, old_value, cache_config_page_size(&cfg->super)); - cache_mark_dirty(cc, old_page); clockcache *clock = (clockcache *)cc; uint64 initial_generation = @@ -528,7 +527,6 @@ test_cache_writeback_fence(cache *cc, memset(new_page->data, new_value, cache_config_page_size(&cfg->super)); - cache_mark_dirty(cc, new_page); cache_unlock(cc, new_page); new_locked = FALSE; cache_unclaim(cc, new_page); @@ -924,7 +922,6 @@ test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) } } for (i = 0; i < cfg->page_capacity; i++) { - cache_mark_dirty(cc, page_arr[i]); cache_unlock(cc, page_arr[i]); cache_unclaim(cc, page_arr[i]); cache_unget(cc, page_arr[i]); @@ -1110,7 +1107,6 @@ cache_test_dirty_flush(cache *cc, rc = STATUS_TEST_FAILED; break; } - cache_mark_dirty(cc, ph); cache_unlock(cc, ph); cache_unclaim(cc, ph); cache_unget(cc, ph); From 7aebf2c8ad61f5e108a1dcaa9d3ffaf9d501619c Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 16:44:35 -0700 Subject: [PATCH 04/13] fix cach writeback impl Signed-off-by: Rob Johnson --- src/clockcache.c | 317 ++++++---------------------------- src/clockcache.h | 15 +- tests/functional/cache_test.c | 169 ++++++++---------- 3 files changed, 130 insertions(+), 371 deletions(-) diff --git a/src/clockcache.c b/src/clockcache.c index 44474e237..10ceb2e71 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -141,9 +141,6 @@ clockcache_print(platform_log_handle *log_handle, clockcache *cc); #define CC_LOADING (1u << 4) // page is actively being read from disk #define CC_WRITELOCKED (1u << 5) // write lock is held #define CC_CLAIMED (1u << 6) // claim is held -// A checkpoint fence has selected this dirty interval for writeback. New -// writers must not claim the page until that writeback completes. -#define CC_WRITEBACK_REQUESTED (1u << 7) /* Common status flag combinations */ // free entry @@ -227,66 +224,25 @@ clockcache_test_flag(clockcache *cc, uint32 entry_number, entry_status flag) /* *-------------------------------------------------------------------------- - * Dirty-generation bookkeeping - * - * A dirty generation identifies the clean->dirty interval, rather than every - * individual mutation. A checkpoint fence rotates the current generation and - * drains all older intervals. Writers cannot modify a page during writeback, - * so writing a later version of an older dirty interval still satisfies the - * fence. + * clockcache_dirty_complete_writeback -- + * + * Mark an entry clean once its writeback has completed. CC_CLEAN must be set + * before CC_WRITEBACK is cleared: the intermediate CC_CLEAN|CC_WRITEBACK state + * is not cleanable, so no thread can start a duplicate writeback in the gap. A + * write lock cannot be held while CC_WRITEBACK is set (see + * clockcache_get_write), so the clean<->dirty transitions never race and no + * lock is needed here. *-------------------------------------------------------------------------- */ -static void -clockcache_dirty_begin(clockcache *cc, uint32 entry_number) -{ - platform_status rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { - clockcache_entry *entry = clockcache_get_entry(cc, entry_number); - debug_assert(entry->dirty_generation == 0); - entry->dirty_generation = - __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); - clockcache_clear_flag(cc, entry_number, CC_CLEAN); - } - - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); -} - static void clockcache_dirty_complete_writeback(clockcache *cc, uint32 entry_number) { - platform_status rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - clockcache_entry *entry = clockcache_get_entry(cc, entry_number); - debug_assert(entry->dirty_generation != 0); - entry->dirty_generation = 0; debug_only uint32 was_clean = clockcache_set_flag(cc, entry_number, CC_CLEAN); debug_assert(!was_clean); debug_only uint32 was_writeback = clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); debug_assert(was_writeback); - clockcache_clear_flag(cc, entry_number, CC_WRITEBACK_REQUESTED); - - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); -} - -static void -clockcache_dirty_discard(clockcache *cc, uint32 entry_number) -{ - platform_status rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - clockcache_entry *entry = clockcache_get_entry(cc, entry_number); - entry->dirty_generation = 0; - clockcache_clear_flag(cc, entry_number, CC_WRITEBACK_REQUESTED); - - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); } #ifdef RECORD_ACQUISITION_STACKS @@ -694,27 +650,11 @@ clockcache_try_get_claim(clockcache *cc, uint32 entry_number) entry_number, clockcache_test_flag(cc, entry_number, CC_CLAIMED)); - if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK_REQUESTED)) { - return GET_RC_CONFLICT; - } - if (clockcache_set_flag(cc, entry_number, CC_CLAIMED)) { clockcache_log(0, entry_number, "return false\n", NULL); return GET_RC_CONFLICT; } - /* - * A fence may have selected the dirty interval between the first test and - * the claim. Do not let a new writer keep that selected interval dirty; - * drop the claim and let the requested writeback proceed instead. - */ - if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK_REQUESTED)) { - debug_only uint32 was_claimed = - clockcache_clear_flag(cc, entry_number, CC_CLAIMED); - debug_assert(was_claimed); - return GET_RC_CONFLICT; - } - clockcache_record_backtrace(cc, entry_number); return GET_RC_SUCCESS; @@ -865,16 +805,11 @@ clockcache_try_get_write(clockcache *cc, uint32 entry_number) static inline bool32 clockcache_ok_to_writeback(clockcache *cc, uint32 entry_number, - bool32 with_access, - bool32 requested_only) + bool32 with_access) { uint32 status = clockcache_get_status(cc, entry_number); - uint32 request_flag = status & CC_WRITEBACK_REQUESTED; - uint32 base_status = status & ~CC_WRITEBACK_REQUESTED; - - return ((!requested_only || request_flag) - && ((base_status == CC_CLEANABLE1_STATUS) - || (with_access && base_status == CC_CLEANABLE2_STATUS))); + return (status == CC_CLEANABLE1_STATUS) + || (with_access && status == CC_CLEANABLE2_STATUS); } /* @@ -900,24 +835,14 @@ clockcache_try_set_writeback(clockcache *cc, volatile uint32 *status = &cc->entry[entry_number].status; if (__sync_bool_compare_and_swap( - status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS) - || __sync_bool_compare_and_swap(status, - CC_CLEANABLE1_STATUS - | CC_WRITEBACK_REQUESTED, - CC_WRITEBACK1_STATUS - | CC_WRITEBACK_REQUESTED)) + status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) { return TRUE; } if (with_access - && (__sync_bool_compare_and_swap( - status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS) - || __sync_bool_compare_and_swap(status, - CC_CLEANABLE2_STATUS - | CC_WRITEBACK_REQUESTED, - CC_WRITEBACK2_STATUS - | CC_WRITEBACK_REQUESTED))) + && __sync_bool_compare_and_swap( + status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) { return TRUE; } @@ -1016,10 +941,7 @@ clockcache_abort_writeback_range(clockcache *cc, *---------------------------------------------------------------------- */ static platform_status -clockcache_batch_start_writeback(clockcache *cc, - uint64 batch, - bool32 is_urgent, - bool32 requested_only) +clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) { uint32 entry_no, next_entry_no; uint64 addr, first_addr, end_addr, i; @@ -1050,8 +972,7 @@ clockcache_batch_start_writeback(clockcache *cc, entry = &cc->entry[entry_no]; addr = entry->page.disk_addr; // test and test and set in the if condition - if (clockcache_ok_to_writeback( - cc, entry_no, is_urgent, requested_only) + if (clockcache_ok_to_writeback(cc, entry_no, is_urgent) && clockcache_try_set_writeback(cc, entry_no, is_urgent)) { debug_assert(clockcache_lookup(cc, addr) == entry_no); @@ -1066,8 +987,7 @@ clockcache_batch_start_writeback(clockcache *cc, next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY - && clockcache_ok_to_writeback( - cc, next_entry_no, is_urgent, requested_only) + && clockcache_ok_to_writeback(cc, next_entry_no, is_urgent) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); first_addr += page_size; end_addr = entry->page.disk_addr; @@ -1081,8 +1001,7 @@ clockcache_batch_start_writeback(clockcache *cc, next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY - && clockcache_ok_to_writeback( - cc, next_entry_no, is_urgent, requested_only) + && clockcache_ok_to_writeback(cc, next_entry_no, is_urgent) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); @@ -1170,100 +1089,21 @@ clockcache_batch_start_writeback(clockcache *cc, return result; } -/* - *-------------------------------------------------------------------------- - * clockcache_fence_request_old_dirty -- - * - * Mark every dirty interval at or before cutoff for writeback. The dirty lock - * makes this atomic with both a clean->dirty transition and a writeback - * completion, so a false return means no interval from this fence remains. - *-------------------------------------------------------------------------- - */ -static bool32 -clockcache_fence_request_old_dirty(clockcache *cc, uint64 cutoff) -{ - bool32 pending = FALSE; - platform_status rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { - clockcache_entry *entry = clockcache_get_entry(cc, entry_no); - if (entry->dirty_generation != 0 - && entry->dirty_generation <= cutoff) - { - clockcache_set_flag(cc, entry_no, CC_WRITEBACK_REQUESTED); - pending = TRUE; - } - } - - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); - return pending; -} - -/* - *-------------------------------------------------------------------------- - * clockcache_fence_cancel_requests -- - * - * Abandon a failed fence without leaving writers permanently blocked behind - * its request bits. A later fence will select these still-dirty generations - * again. - *-------------------------------------------------------------------------- - */ -static void -clockcache_fence_cancel_requests(clockcache *cc, uint64 cutoff) -{ - platform_status rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { - clockcache_entry *entry = clockcache_get_entry(cc, entry_no); - if (entry->dirty_generation != 0 - && entry->dirty_generation <= cutoff) - { - clockcache_clear_flag(cc, entry_no, CC_WRITEBACK_REQUESTED); - } - } - - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); -} - -/* - *-------------------------------------------------------------------------- - * clockcache_fence_start_writeback_round -- - * - * Start writeback only for intervals selected by a checkpoint fence. This is - * a metadata scan of resident cache entries; no page lookup or trunk walk is - * involved. The normal clock hand may retain batch_busy while it owns a free - * batch, so a fence intentionally does not wait for that batch-level state. - * Per-entry compare-and-swap on CC_WRITEBACK provides the required exclusion - * from the normal cleaner. - *-------------------------------------------------------------------------- - */ -static platform_status -clockcache_fence_start_writeback_round(clockcache *cc) -{ - for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { - platform_status rc = - clockcache_batch_start_writeback(cc, batch, TRUE, TRUE); - if (!SUCCESS(rc)) { - return rc; - } - } - return STATUS_OK; -} - /* *-------------------------------------------------------------------------- * clockcache_writeback_fence -- * - * Rotate the dirty generation, then wait only for dirty intervals that were - * already present at that cut. Pages dirtied after the cut carry the next - * generation and do not delay this checkpoint. A request bit blocks new - * writers from indefinitely extending a selected interval, while a writer - * that was already active is allowed to finish and its final bytes are what - * gets written. + * Issue writeback for every dirty, unlocked page and wait for all of it to + * complete, so that a subsequent durable_barrier can make it durable. + * + * A single pass over the cache suffices. Everything reachable from a published + * checkpoint root is copy-on-write and therefore never write-locked while it is + * reachable, so a dirty page that a writer currently holds locked cannot be + * part of this checkpoint and is safely skipped (clockcache_ok_to_writeback + * rejects locked pages). Pages that a concurrent cleaner already has in + * writeback are not re-issued here but are still drained by the final + * io_wait_all. The fence lock serializes checkpoints so that only one thread + * drains the shared I/O contexts at a time. *-------------------------------------------------------------------------- */ platform_status @@ -1272,31 +1112,21 @@ clockcache_writeback_fence(clockcache *cc) platform_status rc = platform_mutex_lock(&cc->writeback_fence_lock); platform_assert_status_ok(rc); - rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - uint64 cutoff = __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); - platform_assert(cutoff < UINT64_MAX); - __atomic_store_n(&cc->dirty_generation, cutoff + 1, __ATOMIC_RELEASE); - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - while (clockcache_fence_request_old_dirty(cc, cutoff)) { - rc = clockcache_fence_start_writeback_round(cc); - if (!SUCCESS(rc)) { - clockcache_fence_cancel_requests(cc, cutoff); + platform_status result = STATUS_OK; + for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { + result = clockcache_batch_start_writeback(cc, batch, TRUE); + if (!SUCCESS(result)) { break; } - /* - * Poll the local completion queue, then yield briefly if all selected - * pages are currently write-locked or in another cleaner's I/O. - */ - clockcache_wait(cc); - platform_sleep_ns(1000); } + // Drain everything issued above (and any in-flight cleaner writeback) so the + // caller's durable_barrier sees completed writes. + io_wait_all(cc->io); + platform_status unlock_rc = platform_mutex_unlock(&cc->writeback_fence_lock); platform_assert_status_ok(unlock_rc); - return rc; + return result; } /* @@ -1397,7 +1227,6 @@ clockcache_try_evict(clockcache *cc, uint32 entry_number) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); entry->type = PAGE_TYPE_INVALID; - clockcache_dirty_discard(cc, entry_number); entry->status = CC_FREE_STATUS; clockcache_log( addr, entry_number, "evict: entry %u addr %lu\n", entry_number, addr); @@ -1478,8 +1307,8 @@ clockcache_move_hand(clockcache *cc, bool32 is_urgent) cleaner_hand = (evict_hand + cc->cleaner_gap) % cc->cfg->batch_capacity; clean_batch_busy = &cc->batch_busy[cleaner_hand]; if (__sync_bool_compare_and_swap(clean_batch_busy, FALSE, TRUE)) { - platform_status rc = clockcache_batch_start_writeback( - cc, cleaner_hand, is_urgent, FALSE); + platform_status rc = + clockcache_batch_start_writeback(cc, cleaner_hand, is_urgent); if (!SUCCESS(rc)) { platform_error_log("clockcache_move_hand: writeback failed: %s\n", platform_status_to_string(rc)); @@ -1506,8 +1335,7 @@ clockcache_get_free_page(clockcache *cc, uint32 status, page_type type, bool32 refcount, - bool32 blocking, - bool32 begins_dirty) + bool32 blocking) { uint32 entry_no; uint64 num_passes = 0; @@ -1517,7 +1345,6 @@ clockcache_get_free_page(clockcache *cc, timestamp wait_start; debug_assert((tid < MAX_THREADS), "Invalid tid=%lu\n", tid); - debug_assert(!begins_dirty || status == CC_ALLOC_STATUS); if (cc->per_thread[tid].free_hand == CC_UNMAPPED_ENTRY) { clockcache_move_hand(cc, FALSE); } @@ -1533,40 +1360,15 @@ clockcache_get_free_page(clockcache *cc, for (entry_no = start_entry; entry_no < end_entry; entry_no++) { entry = &cc->entry[entry_no]; if (entry->status == CC_FREE_STATUS) { - platform_status rc = STATUS_OK; - if (begins_dirty) { - rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - } - bool32 reserved = __sync_bool_compare_and_swap( &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS); if (!reserved) { - if (begins_dirty) { - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); - } continue; } - /* - * The allocation's dirty interval is tagged while holding the - * same lock as a fence cut. This prevents a checkpoint from - * seeing a dirty CC_ALLOC_STATUS entry with generation zero. - */ - debug_assert(entry->dirty_generation == 0); - if (begins_dirty) { - entry->dirty_generation = - __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); - } entry->status = status; entry->type = type; - if (begins_dirty) { - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); - } - if (refcount) { clockcache_inc_ref(cc, entry_no, tid); } @@ -1630,7 +1432,7 @@ clockcache_flush(clockcache *cc) flush_hand++) { platform_status rc = - clockcache_batch_start_writeback(cc, flush_hand, TRUE, FALSE); + clockcache_batch_start_writeback(cc, flush_hand, TRUE); platform_assert_status_ok(rc); } @@ -1692,8 +1494,7 @@ clockcache_alloc(clockcache *cc, uint64 addr, page_type type) CC_ALLOC_STATUS, type, TRUE, // refcount - TRUE, // blocking - TRUE); // begins_dirty + TRUE); // blocking clockcache_entry *entry = &cc->entry[entry_no]; entry->page.disk_addr = addr; entry->type = type; @@ -1793,7 +1594,6 @@ clockcache_try_page_discard(clockcache *cc, uint64 addr) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); entry->type = PAGE_TYPE_INVALID; - clockcache_dirty_discard(cc, entry_number); entry->status = CC_FREE_STATUS; /* 7. reset pincount */ @@ -1928,8 +1728,7 @@ clockcache_acquire_entry_for_load(clockcache *cc, // IN CC_READ_LOADING_STATUS, type, TRUE, // refcount - TRUE, // blocking - FALSE); // begins_dirty + TRUE); // blocking clockcache_entry *entry = clockcache_get_entry(cc, entry_number); /* * If someone else is loading the page and has reserved the lookup, let them @@ -2457,7 +2256,9 @@ clockcache_lock(clockcache *cc, page_handle *page) entry_number, page->disk_addr); clockcache_get_write(cc, entry_number); - clockcache_dirty_begin(cc, entry_number); + // A write lock marks the page dirty; the CC_WRITEBACK exclusion in + // clockcache_get_write guarantees this cannot race a writeback completion. + clockcache_clear_flag(cc, entry_number, CC_CLEAN); } void @@ -2886,7 +2687,7 @@ clockcache_prefetch_pages(clockcache *cc, case GET_RC_EVICTED: { uint32 free_entry_no = clockcache_get_free_page( - cc, CC_READ_LOADING_STATUS, type, FALSE, TRUE, FALSE); + cc, CC_READ_LOADING_STATUS, type, FALSE, TRUE); clockcache_entry *entry = &cc->entry[free_entry_no]; entry->page.disk_addr = addr; entry->type = type; @@ -3700,24 +3501,15 @@ clockcache_init(clockcache *cc, // OUT cc->io = io; cc->heap_id = hid; - platform_status rc = platform_mutex_init(&cc->dirty_lock, mid, hid); - if (!SUCCESS(rc)) { - platform_error_log("clockcache_init: failed to initialize dirty lock: %s\n", - platform_status_to_string(rc)); - goto alloc_error; - } - - rc = platform_mutex_init(&cc->writeback_fence_lock, mid, hid); + platform_status rc = + platform_mutex_init(&cc->writeback_fence_lock, mid, hid); if (!SUCCESS(rc)) { platform_error_log( "clockcache_init: failed to initialize writeback fence lock: %s\n", platform_status_to_string(rc)); - platform_status destroy_rc = platform_mutex_destroy(&cc->dirty_lock); - platform_assert_status_ok(destroy_rc); goto alloc_error; } - cc->dirty_generation = 1; - cc->dirty_locks_initialized = TRUE; + cc->writeback_fence_lock_initialized = TRUE; /* lookup maps addrs to entries, entry contains the entries themselves */ rc = platform_buffer_init(&cc->lookup_bh, @@ -3762,7 +3554,6 @@ clockcache_init(clockcache *cc, // OUT cc->data + clockcache_multiply_by_page_size(cc, i); cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; cc->entry[i].status = CC_FREE_STATUS; - cc->entry[i].dirty_generation = 0; cc->entry[i].type = PAGE_TYPE_INVALID; async_wait_queue_init(&cc->entry[i].waiters); } @@ -3845,12 +3636,10 @@ clockcache_deinit(clockcache *cc) // IN/OUT #endif } - if (cc->dirty_locks_initialized) { + if (cc->writeback_fence_lock_initialized) { rc = platform_mutex_destroy(&cc->writeback_fence_lock); platform_assert_status_ok(rc); - rc = platform_mutex_destroy(&cc->dirty_lock); - platform_assert_status_ok(rc); - cc->dirty_locks_initialized = FALSE; + cc->writeback_fence_lock_initialized = FALSE; } if (cc->lookup) { diff --git a/src/clockcache.h b/src/clockcache.h index 1e601bd54..0ef394a4b 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -73,9 +73,6 @@ typedef uint32 entry_status; // Saved in clockcache_entry->status struct clockcache_entry { page_handle page; volatile entry_status status; - // Generation in which this page's current dirty interval began. Zero means - // no dirty interval (a clean page or an unpublished clean-load reservation). - volatile uint64 dirty_generation; page_type type; async_wait_queue waiters; #ifdef RECORD_ACQUISITION_STACKS @@ -143,16 +140,10 @@ struct clockcache { volatile bool32 *batch_busy; // Convenience pointer for batch_bh uint64 cleaner_gap; - /* - * Dirty-generation bookkeeping for checkpoint writeback fences. The dirty - * lock serializes the clean<->dirty transition with a fence cut and with - * writeback completion. Fence rounds are serialized separately so a page - * only needs one outstanding writeback-request bit. - */ - platform_mutex dirty_lock; + // Serializes checkpoint writeback fences so only one thread drains the + // shared I/O contexts at a time (see clockcache_writeback_fence). platform_mutex writeback_fence_lock; - uint64 dirty_generation; - bool32 dirty_locks_initialized; + bool32 writeback_fence_lock_initialized; volatile struct { volatile uint32 free_hand; diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index 5eb6dd72f..f6ce1d75f 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -411,9 +411,6 @@ typedef struct { platform_status status; } cache_fence_test_context; -/* Mirrors clockcache's private CC_WRITEBACK_REQUESTED status bit. */ -#define CACHE_TEST_WRITEBACK_REQUESTED (1u << 7) - static void cache_test_writeback_fence_thread(void *arg) { @@ -423,10 +420,11 @@ cache_test_writeback_fence_thread(void *arg) } /* - * Verify that a fence drains an already-dirty page without waiting for a page - * dirtied after its cut. Keeping old_page write-locked forces the fence to - * remain in progress long enough to deterministically dirty new_page after - * the dirty-generation rotation. + * Verify the simplified writeback-fence contract: a single pass flushes every + * dirty, unlocked page and completes even while another page is write-locked, + * skipping the locked page rather than waiting for it. old_page is left dirty + * and unlocked so the fence must flush it; new_page is held write-locked so the + * fence must skip it and still finish promptly. */ static platform_status test_cache_writeback_fence(cache *cc, @@ -439,7 +437,6 @@ test_cache_writeback_fence(cache *cc, uint64 *addr_arr = NULL; page_handle *old_page = NULL; page_handle *new_page = NULL; - bool32 old_claimed = FALSE, old_locked = FALSE; bool32 new_claimed = FALSE, new_locked = FALSE; bool32 fence_thread_started = FALSE; const uint8 old_baseline = 0x11, old_value = 0x22; @@ -478,21 +475,30 @@ test_cache_writeback_fence(cache *cc, } cache_flush(cc); + /* Dirty old_page and leave it unlocked: the fence must flush it. */ old_page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); if (!cache_try_claim(cc, old_page)) { rc = STATUS_TEST_FAILED; goto cleanup; } - old_claimed = TRUE; cache_lock(cc, old_page); - old_locked = TRUE; - memset(old_page->data, - old_value, - cache_config_page_size(&cfg->super)); - - clockcache *clock = (clockcache *)cc; - uint64 initial_generation = - __atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE); + memset(old_page->data, old_value, cache_config_page_size(&cfg->super)); + cache_unlock(cc, old_page); + cache_unclaim(cc, old_page); + cache_unget(cc, old_page); + old_page = NULL; + + /* Hold new_page write-locked: the fence must skip it and still complete. */ + new_page = cache_get(cc, addr_arr[pages_per_extent], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, new_page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + new_claimed = TRUE; + cache_lock(cc, new_page); + new_locked = TRUE; + memset(new_page->data, new_value, cache_config_page_size(&cfg->super)); + rc = platform_thread_create(&fence_thread, FALSE, cache_test_writeback_fence_thread, @@ -503,43 +509,18 @@ test_cache_writeback_fence(cache *cc, } fence_thread_started = TRUE; + /* The fence must finish without waiting for the write-locked new_page. */ timestamp wait_start = platform_get_timestamp(); - while (__atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE) - == initial_generation) - { + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { - platform_error_log("cache_test: writeback fence did not take a cut\n"); + platform_error_log( + "cache_test: fence did not complete while a page was locked\n"); rc = STATUS_TIMEDOUT; goto cleanup; } platform_sleep_ns(1000); } - new_page = cache_get( - cc, addr_arr[pages_per_extent], TRUE, PAGE_TYPE_MISC); - if (!cache_try_claim(cc, new_page)) { - rc = STATUS_TEST_FAILED; - goto cleanup; - } - new_claimed = TRUE; - cache_lock(cc, new_page); - new_locked = TRUE; - memset(new_page->data, - new_value, - cache_config_page_size(&cfg->super)); - cache_unlock(cc, new_page); - new_locked = FALSE; - cache_unclaim(cc, new_page); - new_claimed = FALSE; - cache_unget(cc, new_page); - new_page = NULL; - - if (__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { - platform_error_log("cache_test: fence completed while old page was locked\n"); - rc = STATUS_TEST_FAILED; - goto cleanup; - } - cleanup: if (new_locked) { cache_unlock(cc, new_page); @@ -550,12 +531,6 @@ test_cache_writeback_fence(cache *cc, if (new_page != NULL) { cache_unget(cc, new_page); } - if (old_locked) { - cache_unlock(cc, old_page); - } - if (old_claimed) { - cache_unclaim(cc, old_page); - } if (old_page != NULL) { cache_unget(cc, old_page); } @@ -569,10 +544,11 @@ test_cache_writeback_fence(cache *cc, } } + /* Only new_page, write-locked throughout the fence, remains dirty. */ if (SUCCESS(rc)) { uint32 dirty_count = cache_count_dirty(cc); if (dirty_count != 1) { - platform_error_log("cache_test: expected one post-cut dirty page, " + platform_error_log("cache_test: expected one dirty page after fence, " "found %u\n", dirty_count); rc = STATUS_TEST_FAILED; @@ -581,14 +557,15 @@ test_cache_writeback_fence(cache *cc, if (SUCCESS(rc)) { rc = cache_durable_barrier(cc); } + /* The dirty, unlocked page was flushed by the fence. */ if (SUCCESS(rc)) { rc = cache_test_verify_disk_page(cc, addr_arr[0], cache_config_page_size(&cfg->super), old_value); } + /* The write-locked page was skipped, so its baseline is still on disk. */ if (SUCCESS(rc)) { - /* The post-cut page remains dirty, so its baseline must still be disk. */ rc = cache_test_verify_disk_page(cc, addr_arr[pages_per_extent], cache_config_page_size(&cfg->super), @@ -618,8 +595,9 @@ test_cache_writeback_fence(cache *cc, } /* - * A fence must allow an already-claimed writer to finish its selected dirty - * interval, but it must not complete before that writer releases the claim. + * A page that is only claimed (not write-locked) is not cleanable, so a fence + * skips it and completes; the page's baseline is therefore still what is on + * disk. Once the claim is dropped, a second fence flushes the page. */ static platform_status test_cache_writeback_fence_claimed_page(cache *cc, @@ -657,6 +635,7 @@ test_cache_writeback_fence_claimed_page(cache *cc, } cache_flush(cc); + /* Dirty the page, then keep only a claim (release the write lock). */ page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); if (!cache_try_claim(cc, page)) { rc = STATUS_TEST_FAILED; @@ -669,9 +648,6 @@ test_cache_writeback_fence_claimed_page(cache *cc, cache_unlock(cc, page); locked = FALSE; - clockcache *clock = (clockcache *)cc; - uint64 initial_generation = - __atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE); rc = platform_thread_create(&fence_thread, FALSE, cache_test_writeback_fence_thread, @@ -682,41 +658,58 @@ test_cache_writeback_fence_claimed_page(cache *cc, } fence_thread_started = TRUE; - uint64 entry_no = - clock->lookup[addr_arr[0] >> clock->cfg->log_page_size]; - platform_assert(entry_no < clock->cfg->page_capacity); + /* A claimed page is not cleanable, so the fence skips it and completes. */ timestamp wait_start = platform_get_timestamp(); - while ((__atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE) - == initial_generation) - || !(__atomic_load_n(&clock->entry[entry_no].status, - __ATOMIC_ACQUIRE) - & CACHE_TEST_WRITEBACK_REQUESTED)) - { + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { - platform_error_log("cache_test: fence did not request claimed page\n"); + platform_error_log( + "cache_test: fence did not complete while page was claimed\n"); rc = STATUS_TIMEDOUT; goto cleanup; } platform_sleep_ns(1000); } - if (__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { - platform_error_log("cache_test: fence completed while page was claimed\n"); - rc = STATUS_TEST_FAILED; + platform_status join_rc = platform_thread_join(&fence_thread); + fence_thread_started = FALSE; + if (!SUCCESS(join_rc)) { + rc = join_rc; + goto cleanup; + } + if (!SUCCESS(fence_ctxt.status)) { + rc = fence_ctxt.status; goto cleanup; } - /* This claim predates the request, so it is permitted to finish. */ - cache_lock(cc, page); - locked = TRUE; - memset(page->data, final_value, cache_config_page_size(&cfg->super)); - cache_unlock(cc, page); - locked = FALSE; + /* The skipped page was not written, so its baseline is still on disk. */ + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* Drop the claim; the page is now cleanable and a second fence flushes it. */ cache_unclaim(cc, page); claimed = FALSE; cache_unget(cc, page); page = NULL; + rc = cache_writeback_fence(cc); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (cache_count_dirty(cc) != 0) { + platform_error_log("cache_test: second fence left dirty pages\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = cache_durable_barrier(cc); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), final_value); + cleanup: if (locked) { cache_unlock(cc, page); @@ -728,29 +721,15 @@ test_cache_writeback_fence_claimed_page(cache *cc, cache_unget(cc, page); } if (fence_thread_started) { - platform_status join_rc = platform_thread_join(&fence_thread); - if (!SUCCESS(join_rc) && SUCCESS(rc)) { - rc = join_rc; + platform_status jrc = platform_thread_join(&fence_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; } if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { rc = fence_ctxt.status; } } - if (SUCCESS(rc) && cache_count_dirty(cc) != 0) { - platform_error_log("cache_test: claimed-page fence left dirty pages\n"); - rc = STATUS_TEST_FAILED; - } - if (SUCCESS(rc)) { - rc = cache_durable_barrier(cc); - } - if (SUCCESS(rc)) { - rc = cache_test_verify_disk_page(cc, - addr_arr[0], - cache_config_page_size(&cfg->super), - final_value); - } - if (addr_arr != NULL) { cache_flush(cc); allocator *al = cache_get_allocator(cc); From c8d0fd8ff5f58482f375f8695970324fbe43094f Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 23:00:50 -0700 Subject: [PATCH 05/13] simplify cache writeback Signed-off-by: Rob Johnson --- src/cache.h | 23 ++- src/clockcache.c | 267 ++++++++++++++++++++------------ src/clockcache.h | 13 +- src/core.c | 178 +++++++++++----------- src/shard_log.c | 2 +- src/trunk.c | 2 +- tests/functional/cache_test.c | 277 ++++++++++++++++++++++++++-------- tests/functional/log_test.c | 102 +++++-------- 8 files changed, 530 insertions(+), 334 deletions(-) diff --git a/src/cache.h b/src/cache.h index 1cecd1318..8e37827b2 100644 --- a/src/cache.h +++ b/src/cache.h @@ -98,7 +98,7 @@ typedef void (*cache_generic_fn)(cache *cc); typedef uint64 (*cache_generic_uint64_fn)(cache *cc); typedef void (*page_generic_fn)(cache *cc, page_handle *page); typedef platform_status (*cache_durable_barrier_fn)(cache *cc); -typedef platform_status (*cache_writeback_fence_fn)(cache *cc); +typedef platform_status (*cache_writeback_dirty_fn)(cache *cc); typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); @@ -177,7 +177,7 @@ typedef struct cache_ops { page_writeback_fn page_writeback; extent_writeback_fn extent_writeback; cache_generic_fn flush; - cache_writeback_fence_fn writeback_fence; + cache_writeback_dirty_fn writeback_dirty; cache_durable_barrier_fn durable_barrier; evict_fn evict; cache_generic_fn cleanup; @@ -369,11 +369,7 @@ cache_unclaim(cache *cc, page_handle *page) * * Blocks until outstanding read locks are released by other threads. * - * Acquiring the write lock marks the page dirty: it begins a dirty interval on - * this transition (before the caller's first change), so a checkpoint fence - * cannot miss the modification. A page obtained write-locked from cache_alloc() - * is likewise already dirty. Callers therefore do not separately declare the - * mutation. + * Acquiring the write lock marks the page dirty. *---------------------------------------------------------------------- */ static inline void @@ -536,17 +532,16 @@ cache_flush(cache *cc) /* *----------------------------------------------------------------------------- - * cache_writeback_fence + * cache_writeback_dirty * - * Wait until every page whose current dirty interval began before this call's - * cut has completed writeback. Pages dirtied after the cut do not delay the - * call. + * Issues and wait for completion of writebacks for all pages that are dirty + * but not locked at the time of the call. May writeback other pages, as well. *----------------------------------------------------------------------------- */ static inline platform_status -cache_writeback_fence(cache *cc) +cache_writeback_dirty(cache *cc) { - return cc->ops->writeback_fence(cc); + return cc->ops->writeback_dirty(cc); } /* @@ -554,7 +549,7 @@ cache_writeback_fence(cache *cc) * cache_durable_barrier * * Ensure that writeback completed before this call is durable across a power - * loss. Callers normally use this after cache_writeback_fence(), and again + * loss. Callers normally use this after cache_writeback_dirty(), and again * after publishing a checkpoint superblock. *----------------------------------------------------------------------------- */ diff --git a/src/clockcache.c b/src/clockcache.c index 10ceb2e71..cfa48e346 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -224,19 +224,40 @@ clockcache_test_flag(clockcache *cc, uint32 entry_number, entry_status flag) /* *-------------------------------------------------------------------------- - * clockcache_dirty_complete_writeback -- - * + * Dirty-generation bookkeeping + * + * Each entry records the generation in which its current dirty interval began + * (0 when clean/free). A writeback fence takes a "cut" of the generation + * counter and drains every entry stamped at or below that cut. These + * transitions need no dedicated lock: clockcache_dirty_begin runs under the + * page's write lock, and a write lock cannot be held while CC_WRITEBACK is set + * (see clockcache_get_write), so a clean->dirty transition never races a + * writeback completion on the same entry. + *-------------------------------------------------------------------------- + */ +static void +clockcache_dirty_begin(clockcache *cc, uint32 entry_number) +{ + if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + debug_assert(entry->dirty_generation == 0); + // Stamp before clearing CC_CLEAN so a fence that observes the page dirty + // always sees a valid generation. clear_flag is a full barrier. + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + clockcache_clear_flag(cc, entry_number, CC_CLEAN); + } +} + +/* * Mark an entry clean once its writeback has completed. CC_CLEAN must be set * before CC_WRITEBACK is cleared: the intermediate CC_CLEAN|CC_WRITEBACK state - * is not cleanable, so no thread can start a duplicate writeback in the gap. A - * write lock cannot be held while CC_WRITEBACK is set (see - * clockcache_get_write), so the clean<->dirty transitions never race and no - * lock is needed here. - *-------------------------------------------------------------------------- + * is not cleanable, so no thread can start a duplicate writeback in the gap. */ static void clockcache_dirty_complete_writeback(clockcache *cc, uint32 entry_number) { + clockcache_get_entry(cc, entry_number)->dirty_generation = 0; debug_only uint32 was_clean = clockcache_set_flag(cc, entry_number, CC_CLEAN); debug_assert(!was_clean); @@ -820,6 +841,13 @@ clockcache_ok_to_writeback(clockcache *cc, * status must be: * -- CC_CLEANABLE1_STATUS (= 0) // dirty * -- CC_CLEANABLE2_STATUS (= 0 | CC_ACCESSED) // dirty + * + * Returns FALSE only if the page is genuinely not writeback-able (locked, + * claimed, already in writeback, clean, ...). The CC_ACCESSED bit can flip + * (a reader sets it, the clock hand clears it) between the two + *compare-and- swaps, so we retry as long as the status remains one of the + *cleanable states rather than spuriously failing on a page that stayed + *cleanable. *---------------------------------------------------------------------- */ static inline bool32 @@ -834,19 +862,27 @@ clockcache_try_set_writeback(clockcache *cc, cc->cfg->page_capacity); volatile uint32 *status = &cc->entry[entry_number].status; - if (__sync_bool_compare_and_swap( - status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) - { - return TRUE; - } - - if (with_access - && __sync_bool_compare_and_swap( - status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) - { - return TRUE; + while (TRUE) { + uint32 cur = *status; + if (cur == CC_CLEANABLE1_STATUS) { + if (__sync_bool_compare_and_swap( + status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) + { + return TRUE; + } + } else if (with_access && cur == CC_CLEANABLE2_STATUS) { + if (__sync_bool_compare_and_swap( + status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) + { + return TRUE; + } + } else { + // Not a cleanable state: the page cannot be written back right now. + return FALSE; + } + // The CAS failed because the status changed under us. If it is still + // cleanable, retry; otherwise the next iteration returns FALSE. } - return FALSE; } typedef struct async_io_state { @@ -1091,42 +1127,72 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) /* *-------------------------------------------------------------------------- - * clockcache_writeback_fence -- - * - * Issue writeback for every dirty, unlocked page and wait for all of it to - * complete, so that a subsequent durable_barrier can make it durable. - * - * A single pass over the cache suffices. Everything reachable from a published - * checkpoint root is copy-on-write and therefore never write-locked while it is - * reachable, so a dirty page that a writer currently holds locked cannot be - * part of this checkpoint and is safely skipped (clockcache_ok_to_writeback - * rejects locked pages). Pages that a concurrent cleaner already has in - * writeback are not re-issued here but are still drained by the final - * io_wait_all. The fence lock serializes checkpoints so that only one thread - * drains the shared I/O contexts at a time. + * clockcache_writeback_dirty -- + * + * Write back every page that was dirty when this call began, and wait for + * those writes (only those) to complete, so a subsequent durable_barrier can + * make them durable. + * + * We take a "cut" by incrementing the dirty generation: pages dirtied before + * now carry a generation <= cutoff and must be drained; pages dirtied + * afterward carry a higher generation and never delay this call. This is what + * lets us avoid io_wait_all, which waits for *global* I/O quiescence -- a + * condition a busy cache (background cleaner writes, async reads) may never + * reach. + * + * A single scan of the entries suffices, because a generation only ever moves + * above the cut (a page must go clean before it can be re-dirtied), so an entry + * that has left the pre-cut set never re-enters it. We first issue writeback + * for every dirty, unlocked page in one bulk pass so the writes pipeline, then + * walk the entries once, waiting per entry for its pre-cut interval to drain. + * + * The bulk pass leaves every pre-cut page that belongs to this checkpoint in + * CC_WRITEBACK: such a page is dirty and, under copy-on-write, never locked or + * claimed, so clockcache_try_set_writeback issues it (or a concurrent cleaner + * already has). The drain therefore only has to wait on pages that are in + * writeback; a pre-cut page that is not in writeback is either already clean, + * or was held by a writer during the bulk pass and so cannot belong to this + * checkpoint -- either way we skip it, which is what keeps this deadlock-free. + * Per-context background reapers complete the issued writes; we also poll our + * own context via clockcache_wait to help things along. + * + * Termination: once a page is in writeback the writer is excluded (see + * clockcache_get_write), so it progresses to clean and, if re-dirtied, moves to + * a generation above the cut. *-------------------------------------------------------------------------- */ platform_status -clockcache_writeback_fence(clockcache *cc) +clockcache_writeback_dirty(clockcache *cc) { - platform_status rc = platform_mutex_lock(&cc->writeback_fence_lock); - platform_assert_status_ok(rc); + uint64 cutoff = + __atomic_fetch_add(&cc->dirty_generation, 1, __ATOMIC_SEQ_CST); + platform_assert(cutoff < UINT64_MAX); - platform_status result = STATUS_OK; + // Bulk-issue writeback for every dirty, unlocked page so the writes pipeline + // rather than draining one batch at a time. for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { - result = clockcache_batch_start_writeback(cc, batch, TRUE); + platform_status result = clockcache_batch_start_writeback(cc, batch, TRUE); if (!SUCCESS(result)) { - break; + return result; } } - // Drain everything issued above (and any in-flight cleaner writeback) so the - // caller's durable_barrier sees completed writes. - io_wait_all(cc->io); + // Wait for each pre-cut interval's writeback to complete, in a single pass. + for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_no); + while (TRUE) { + uint64 gen = + __atomic_load_n(&entry->dirty_generation, __ATOMIC_RELAXED); + if (gen == 0 || gen > cutoff + || !clockcache_test_flag(cc, entry_no, CC_WRITEBACK)) + { + break; + } + clockcache_wait(cc); + } + } - platform_status unlock_rc = platform_mutex_unlock(&cc->writeback_fence_lock); - platform_assert_status_ok(unlock_rc); - return result; + return STATUS_OK; } /* @@ -1226,8 +1292,9 @@ clockcache_try_evict(clockcache *cc, uint32 entry_number) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); - entry->type = PAGE_TYPE_INVALID; - entry->status = CC_FREE_STATUS; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; + entry->status = CC_FREE_STATUS; clockcache_log( addr, entry_number, "evict: entry %u addr %lu\n", entry_number, addr); @@ -1366,6 +1433,14 @@ clockcache_get_free_page(clockcache *cc, continue; } + // A page that begins dirty (a fresh allocation) must carry a dirty + // generation, just like a clean->dirty transition. The entry is + // write-locked here, so a concurrent fence skips it regardless. + if (!(status & CC_CLEAN)) { + debug_assert(entry->dirty_generation == 0); + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + } entry->status = status; entry->type = type; @@ -1593,8 +1668,9 @@ clockcache_try_page_discard(clockcache *cc, uint64 addr) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); - entry->type = PAGE_TYPE_INVALID; - entry->status = CC_FREE_STATUS; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; + entry->status = CC_FREE_STATUS; /* 7. reset pincount */ clockcache_reset_pin(cc, entry_number); @@ -1711,8 +1787,9 @@ clockcache_get_in_cache(clockcache *cc, // IN static void clockcache_release_unpublished_entry(clockcache_entry *entry) { - entry->page.disk_addr = CC_UNMAPPED_ADDR; - entry->type = PAGE_TYPE_INVALID; + entry->page.disk_addr = CC_UNMAPPED_ADDR; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; platform_assert(entry->waiters.head == NULL); entry->status = CC_FREE_STATUS; } @@ -2256,9 +2333,10 @@ clockcache_lock(clockcache *cc, page_handle *page) entry_number, page->disk_addr); clockcache_get_write(cc, entry_number); - // A write lock marks the page dirty; the CC_WRITEBACK exclusion in - // clockcache_get_write guarantees this cannot race a writeback completion. - clockcache_clear_flag(cc, entry_number, CC_CLEAN); + // A write lock marks the page dirty (and stamps its dirty generation). The + // CC_WRITEBACK exclusion in clockcache_get_write guarantees this cannot race + // a writeback completion. + clockcache_dirty_begin(cc, entry_number); } void @@ -2373,11 +2451,12 @@ clockcache_page_writeback(clockcache *cc, if (!is_blocking) { state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); if (state == NULL) { - platform_error_log("clockcache_page_writeback: async_io_state allocation " - "failed for addr %lu, entry %u, type %u\n", - addr, - entry_number, - type); + platform_error_log( + "clockcache_page_writeback: async_io_state allocation " + "failed for addr %lu, entry %u, type %u\n", + addr, + entry_number, + type); } platform_assert(state); state->cc = cc; @@ -2389,34 +2468,37 @@ clockcache_page_writeback(clockcache *cc, clockcache_write_callback, state); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_writeback: io_async_state_init failed " - "for addr %lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_async_state_init failed " + "for addr %lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); status = io_async_state_append_page(state->iostate, page->data); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_writeback: io_async_state_append_page " - "failed for addr %lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_async_state_append_page " + "failed for addr %lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); io_async_run(state->iostate); } else { status = io_write(cc->io, page->data, clockcache_page_size(cc), addr); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_writeback: io_write failed for addr " - "%lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_write failed for addr " + "%lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); clockcache_log(addr, @@ -3261,10 +3343,10 @@ clockcache_flush_virtual(cache *c) } platform_status -clockcache_writeback_fence_virtual(cache *c) +clockcache_writeback_dirty_virtual(cache *c) { clockcache *cc = (clockcache *)c; - return clockcache_writeback_fence(cc); + return clockcache_writeback_dirty(cc); } platform_status @@ -3407,7 +3489,7 @@ static cache_ops clockcache_ops = { .page_writeback = clockcache_page_writeback_virtual, .extent_writeback = clockcache_extent_writeback_virtual, .flush = clockcache_flush_virtual, - .writeback_fence = clockcache_writeback_fence_virtual, + .writeback_dirty = clockcache_writeback_dirty_virtual, .durable_barrier = clockcache_durable_barrier_virtual, .evict = clockcache_evict_all_virtual, .cleanup = clockcache_wait_virtual, @@ -3501,18 +3583,12 @@ clockcache_init(clockcache *cc, // OUT cc->io = io; cc->heap_id = hid; - platform_status rc = - platform_mutex_init(&cc->writeback_fence_lock, mid, hid); - if (!SUCCESS(rc)) { - platform_error_log( - "clockcache_init: failed to initialize writeback fence lock: %s\n", - platform_status_to_string(rc)); - goto alloc_error; - } - cc->writeback_fence_lock_initialized = TRUE; + // Generation 0 is the sentinel for "clean/free", so dirty stamping starts + // at 1. + cc->dirty_generation = 1; /* lookup maps addrs to entries, entry contains the entries themselves */ - rc = platform_buffer_init(&cc->lookup_bh, + platform_status rc = platform_buffer_init(&cc->lookup_bh, allocator_page_capacity * sizeof(cc->lookup[0])); if (!SUCCESS(rc)) { platform_error_log("clockcache_init: failed to allocate lookup table " @@ -3552,9 +3628,10 @@ clockcache_init(clockcache *cc, // OUT for (i = 0; i < cc->cfg->page_capacity; i++) { cc->entry[i].page.data = cc->data + clockcache_multiply_by_page_size(cc, i); - cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; - cc->entry[i].status = CC_FREE_STATUS; - cc->entry[i].type = PAGE_TYPE_INVALID; + cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; + cc->entry[i].status = CC_FREE_STATUS; + cc->entry[i].dirty_generation = 0; + cc->entry[i].type = PAGE_TYPE_INVALID; async_wait_queue_init(&cc->entry[i].waiters); } @@ -3636,12 +3713,6 @@ clockcache_deinit(clockcache *cc) // IN/OUT #endif } - if (cc->writeback_fence_lock_initialized) { - rc = platform_mutex_destroy(&cc->writeback_fence_lock); - platform_assert_status_ok(rc); - cc->writeback_fence_lock_initialized = FALSE; - } - if (cc->lookup) { rc = platform_buffer_deinit(&cc->lookup_bh); if (!SUCCESS(rc)) { diff --git a/src/clockcache.h b/src/clockcache.h index 0ef394a4b..421f070ee 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -73,6 +73,10 @@ typedef uint32 entry_status; // Saved in clockcache_entry->status struct clockcache_entry { page_handle page; volatile entry_status status; + // Generation in which this page's current dirty interval began; 0 when the + // page is clean or free. A writeback fence drains every entry whose + // generation is at or below the cut it took (see clockcache_writeback_dirty). + volatile uint64 dirty_generation; page_type type; async_wait_queue waiters; #ifdef RECORD_ACQUISITION_STACKS @@ -140,10 +144,11 @@ struct clockcache { volatile bool32 *batch_busy; // Convenience pointer for batch_bh uint64 cleaner_gap; - // Serializes checkpoint writeback fences so only one thread drains the - // shared I/O contexts at a time (see clockcache_writeback_fence). - platform_mutex writeback_fence_lock; - bool32 writeback_fence_lock_initialized; + // Monotonic generation counter. A writeback fence atomically increments it + // to take a "cut", then drains every entry stamped with a generation at or + // below that cut. Concurrent fences are safe: each takes a distinct cut and + // waits only on its own I/O context, so no lock is needed. + uint64 dirty_generation; volatile struct { volatile uint32 free_hand; diff --git a/src/core.c b/src/core.c index b91d6f2e7..9af1975ab 100644 --- a/src/core.c +++ b/src/core.c @@ -59,9 +59,8 @@ _Static_assert(CORE_NUM_MEMTABLES <= MAX_MEMTABLES, static platform_status core_checkpoint_lock_init(core_handle *spl) { - platform_status rc = platform_mutex_init(&spl->checkpoint_lock, - platform_get_module_id(), - spl->heap_id); + platform_status rc = platform_mutex_init( + &spl->checkpoint_lock, platform_get_module_id(), spl->heap_id); if (SUCCESS(rc)) { spl->checkpoint_lock_initialized = TRUE; } @@ -165,17 +164,17 @@ typedef struct ONDISK core_checkpoint_record { * The highest memtable generation incorporated in root_addr. The boolean * keeps the fresh-database case distinct from generation zero. */ - uint64 incorporated_generation; - bool32 has_incorporated_generation; - uint64 root_addr; - uint64 timestamp; - uint64 sequence; - uint64 table_id; - uint32 record_slot; - bool32 checkpointed; - bool32 unmounted; - uint64 magic; - uint64 format_version; + uint64 incorporated_generation; + bool32 has_incorporated_generation; + uint64 root_addr; + uint64 timestamp; + uint64 sequence; + uint64 table_id; + uint32 record_slot; + bool32 checkpointed; + bool32 unmounted; + uint64 magic; + uint64 format_version; checksum128 checksum; } core_checkpoint_record; @@ -216,7 +215,7 @@ core_checkpoint_record_addr_is_valid(core_handle *spl, uint64 addr) } static bool32 -core_checkpoint_directory_is_valid(core_handle *spl, +core_checkpoint_directory_is_valid(core_handle *spl, const core_checkpoint_directory *directory) { if (directory->magic != CORE_CHECKPOINT_DIRECTORY_MAGIC @@ -228,19 +227,20 @@ core_checkpoint_directory_is_valid(core_handle *spl, return FALSE; } - uint64 record0 = directory->record_addr[0]; - uint64 record1 = directory->record_addr[1]; + uint64 record0 = directory->record_addr[0]; + uint64 record1 = directory->record_addr[1]; allocator_config *allocator_cfg = allocator_get_config(spl->al); return core_checkpoint_record_addr_is_valid(spl, record0) && core_checkpoint_record_addr_is_valid(spl, record1) && record0 != record1 - && !allocator_config_pages_share_extent(allocator_cfg, record0, record1); + && !allocator_config_pages_share_extent( + allocator_cfg, record0, record1); } static bool32 -core_checkpoint_record_is_valid(core_handle *spl, +core_checkpoint_record_is_valid(core_handle *spl, const core_checkpoint_record *record, - uint64 record_slot) + uint64 record_slot) { return record->magic == CORE_CHECKPOINT_RECORD_MAGIC && record->format_version == CORE_CHECKPOINT_FORMAT_VERSION @@ -293,7 +293,7 @@ core_initialize_checkpoint_record_page(core_handle *spl, uint64 page_addr) } static platform_status -core_create_checkpoint_directory(core_handle *spl, +core_create_checkpoint_directory(core_handle *spl, core_checkpoint_directory *directory) { uint64 directory_addr; @@ -338,7 +338,7 @@ core_create_checkpoint_directory(core_handle *spl, * bootstrap mapping through the shared backing I/O handle, which the * following durable barrier fdatasyncs with the directory page. */ - rc = cache_writeback_fence(spl->cc); + rc = cache_writeback_dirty(spl->cc); if (!SUCCESS(rc)) { return rc; } @@ -346,7 +346,7 @@ core_create_checkpoint_directory(core_handle *spl, } static platform_status -core_get_checkpoint_directory(core_handle *spl, +core_get_checkpoint_directory(core_handle *spl, core_checkpoint_directory *directory) { uint64 directory_addr; @@ -371,21 +371,19 @@ core_get_checkpoint_directory(core_handle *spl, } static platform_status -core_load_checkpoint_records(core_handle *spl, +core_load_checkpoint_records(core_handle *spl, const core_checkpoint_directory *directory, - core_checkpoint_records *records) + core_checkpoint_records *records) { ZERO_CONTENTS(records); for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { - page_handle *page = cache_get(spl->cc, - directory->record_addr[slot], - TRUE, - PAGE_TYPE_SUPERBLOCK); + page_handle *page = cache_get( + spl->cc, directory->record_addr[slot], TRUE, PAGE_TYPE_SUPERBLOCK); memcpy(&records->record[slot], page->data, sizeof(records->record[slot])); cache_unget(spl->cc, page); - records->valid[slot] = core_checkpoint_record_is_valid( - spl, &records->record[slot], slot); + records->valid[slot] = + core_checkpoint_record_is_valid(spl, &records->record[slot], slot); if (!records->valid[slot]) { continue; } @@ -422,8 +420,7 @@ core_load_checkpoint_records(core_handle *spl, static void core_destroy_checkpoint_record_extent(core_handle *spl, uint64 record_addr) { - refcount ref = - allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); + refcount ref = allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); if (ref != AL_NO_REFS) { platform_error_log("core_destroy_checkpoint_record_extent: record extent " "%lu has unexpected refcount %u\n", @@ -516,9 +513,8 @@ core_capture_checkpoint_cut(core_handle *spl, } *has_incorporated_generation = retired_generation != UINT64_MAX; - *incorporated_generation = *has_incorporated_generation - ? retired_generation - : 0; + *incorporated_generation = + *has_incorporated_generation ? retired_generation : 0; return STATUS_OK; } @@ -528,11 +524,11 @@ core_publish_checkpoint_record(core_handle *spl, bool32 is_unmount, bool32 is_create) { - uint64 old_root_addr; - platform_status rc; - trunk_snapshot snapshot; - bool32 has_incorporated_generation; - uint64 incorporated_generation; + uint64 old_root_addr; + platform_status rc; + trunk_snapshot snapshot; + bool32 has_incorporated_generation; + uint64 incorporated_generation; core_checkpoint_directory directory; core_checkpoint_records records; uint64 target_slot; @@ -549,10 +545,8 @@ core_publish_checkpoint_record(core_handle *spl, return rc; } - rc = core_capture_checkpoint_cut(spl, - &snapshot, - &has_incorporated_generation, - &incorporated_generation); + rc = core_capture_checkpoint_cut( + spl, &snapshot, &has_incorporated_generation, &incorporated_generation); if (!SUCCESS(rc)) { goto unlock_checkpoint; } @@ -601,32 +595,30 @@ core_publish_checkpoint_record(core_handle *spl, * is durable, then release only the overwritten owner. The newest record * remains independently live as the torn-write fallback. */ - old_root_addr = records.valid[target_slot] - ? records.record[target_slot].root_addr - : 0; + old_root_addr = + records.valid[target_slot] ? records.record[target_slot].root_addr : 0; ZERO_CONTENTS(&record); record.incorporated_generation = incorporated_generation; record.has_incorporated_generation = has_incorporated_generation; - record.root_addr = snapshot.root_addr; - record.timestamp = platform_get_real_time(); - record.sequence = records.have_newest - ? records.record[records.newest_slot].sequence + 1 - : 1; - record.table_id = spl->id; - record.record_slot = target_slot; - record.checkpointed = is_checkpoint; - record.unmounted = is_unmount; - record.magic = CORE_CHECKPOINT_RECORD_MAGIC; - record.format_version = CORE_CHECKPOINT_FORMAT_VERSION; + record.root_addr = snapshot.root_addr; + record.timestamp = platform_get_real_time(); + record.sequence = records.have_newest + ? records.record[records.newest_slot].sequence + 1 + : 1; + record.table_id = spl->id; + record.record_slot = target_slot; + record.checkpointed = is_checkpoint; + record.unmounted = is_unmount; + record.magic = CORE_CHECKPOINT_RECORD_MAGIC; + record.format_version = CORE_CHECKPOINT_FORMAT_VERSION; record.checksum = core_checkpoint_record_checksum(&record); - core_write_checkpoint_page(spl, - directory.record_addr[target_slot], - &record, - sizeof(record)); - /* The record now owns this reference, even if the barrier reports failure. */ + core_write_checkpoint_page( + spl, directory.record_addr[target_slot], &record, sizeof(record)); + /* The record now owns this reference, even if the barrier reports failure. + */ snapshot.root_addr = 0; rc = cache_durable_barrier(spl->cc); @@ -655,21 +647,21 @@ core_publish_checkpoint_record(core_handle *spl, goto unlock_checkpoint; release_snapshot: - { - platform_status release_rc = - trunk_snapshot_release(&spl->trunk_context, &snapshot); - if (SUCCESS(rc) && !SUCCESS(release_rc)) { - rc = release_rc; - } +{ + platform_status release_rc = + trunk_snapshot_release(&spl->trunk_context, &snapshot); + if (SUCCESS(rc) && !SUCCESS(release_rc)) { + rc = release_rc; } +} unlock_checkpoint: - { - platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); - if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { - rc = unlock_rc; - } +{ + platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); + if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { + rc = unlock_rc; } +} return rc; } @@ -2313,14 +2305,15 @@ core_mkfs(core_handle *spl, platform_status rc = core_checkpoint_lock_init(spl); if (!SUCCESS(rc)) { - platform_error_log("core_mkfs: checkpoint lock initialization failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mkfs: checkpoint lock initialization failed: %s\n", + platform_status_to_string(rc)); return rc; } // set up the memtable context memtable_config *mt_cfg = &spl->cfg.mt_cfg; - rc = memtable_context_init(&spl->mt_ctxt, + rc = memtable_context_init(&spl->mt_ctxt, spl->heap_id, cc, mt_cfg, @@ -2359,8 +2352,9 @@ core_mkfs(core_handle *spl, rc = core_publish_checkpoint_record(spl, FALSE, FALSE, TRUE); if (!SUCCESS(rc)) { - platform_error_log("core_mkfs: core_publish_checkpoint_record failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mkfs: core_publish_checkpoint_record failed: %s\n", + platform_status_to_string(rc)); goto deinit_stats; } return STATUS_OK; @@ -2405,8 +2399,9 @@ core_mount(core_handle *spl, platform_status rc = core_checkpoint_lock_init(spl); if (!SUCCESS(rc)) { - platform_error_log("core_mount: checkpoint lock initialization failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mount: checkpoint lock initialization failed: %s\n", + platform_status_to_string(rc)); return rc; } @@ -2416,7 +2411,7 @@ core_mount(core_handle *spl, * unmounted record supplies the root. We still validate both records and * choose the newest clean one by sequence rather than wall-clock time. */ - uint64 root_addr = 0; + uint64 root_addr = 0; bool32 has_incorporated_generation = FALSE; uint64 incorporated_generation = 0; core_checkpoint_directory directory; @@ -2436,8 +2431,7 @@ core_mount(core_handle *spl, rc = STATUS_BAD_PARAM; goto deinit_checkpoint_lock; } - const core_checkpoint_record *record = - &records.record[records.newest_slot]; + const core_checkpoint_record *record = &records.record[records.newest_slot]; if (!record->unmounted) { /* * This is an interrupted run. An older clean record is only an A/B @@ -2450,12 +2444,12 @@ core_mount(core_handle *spl, rc = STATUS_INVALID_STATE; goto deinit_checkpoint_lock; } - root_addr = record->root_addr; + root_addr = record->root_addr; has_incorporated_generation = record->has_incorporated_generation; incorporated_generation = record->incorporated_generation; memtable_config *mt_cfg = &spl->cfg.mt_cfg; - rc = memtable_context_init_at_generation( + rc = memtable_context_init_at_generation( &spl->mt_ctxt, spl->heap_id, cc, @@ -2496,8 +2490,9 @@ core_mount(core_handle *spl, rc = core_publish_checkpoint_record(spl, FALSE, FALSE, FALSE); if (!SUCCESS(rc)) { - platform_error_log("core_mount: core_publish_checkpoint_record failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mount: core_publish_checkpoint_record failed: %s\n", + platform_status_to_string(rc)); goto deinit_stats; } return STATUS_OK; @@ -2626,8 +2621,9 @@ core_unmount(core_handle *spl) core_quiesce_for_shutdown(spl); rc = core_publish_checkpoint_record(spl, FALSE, TRUE, FALSE); if (!SUCCESS(rc)) { - platform_error_log("core_unmount: failed to publish checkpoint record: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_unmount: failed to publish checkpoint record: %s\n", + platform_status_to_string(rc)); } core_teardown_after_shutdown(spl); trunk_context_deinit(&spl->trunk_context); diff --git a/src/shard_log.c b/src/shard_log.c index 205ed3cad..f90963232 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -395,7 +395,7 @@ shard_log_write(log_handle *logh, * In particular, thread_data is otherwise only protected by the * per-thread writer convention, not by a log-wide lock. This function * intentionally does not issue writeback: after establishing that - * exclusion, the caller takes cache_writeback_fence(), followed by a + * exclusion, the caller takes cache_writeback_dirty(), followed by a * durable barrier. */ platform_status diff --git a/src/trunk.c b/src/trunk.c index c95c0c72e..8ebbc0b2b 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -6852,7 +6852,7 @@ trunk_context_clone(trunk_context *dst, trunk_context *src) platform_status trunk_make_durable(trunk_context *context) { - platform_status rc = cache_writeback_fence(context->cc); + platform_status rc = cache_writeback_dirty(context->cc); if (!SUCCESS(rc)) { return rc; } diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index f6ce1d75f..7e4a06b07 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -113,9 +113,9 @@ cache_test_verify_disk_page(cache *cc, uint64 page_size, uint8 expected) { - clockcache *clock = (clockcache *)cc; - buffer_handle buffer; - platform_status rc = platform_buffer_init(&buffer, page_size); + clockcache *clock = (clockcache *)cc; + buffer_handle buffer; + platform_status rc = platform_buffer_init(&buffer, page_size); if (!SUCCESS(rc)) { return rc; } @@ -180,8 +180,8 @@ cache_test_zero_disk_page(io_handle *io, uint64 addr, uint64 page_size) */ static platform_status test_rc_allocator_recovery_bootstrap(allocator_config *cfg, - io_handle *io, - platform_heap_id hid) + io_handle *io, + platform_heap_id hid) { const allocator_root_id root_id = 1; uint64 refcount_buffer_size = @@ -192,28 +192,28 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, uint64 reserved_extent_count = 1 + refcount_extent_count + 2; uint64 old_clean_state_addr = (1 + refcount_extent_count + 1) * cfg->io_cfg->extent_size; - uint64 super_addr = 0; - uint64 stale_extent_addr = 0; - platform_status rc = STATUS_OK; - rc_allocator original, recovery, remounted; - bool32 original_live = FALSE; - bool32 recovery_live = FALSE; - bool32 remounted_live = FALSE; + uint64 super_addr = 0; + uint64 stale_extent_addr = 0; + platform_status rc = STATUS_OK; + rc_allocator original, recovery, remounted; + bool32 original_live = FALSE; + bool32 recovery_live = FALSE; + bool32 remounted_live = FALSE; ZERO_CONTENTS(&original); ZERO_CONTENTS(&recovery); ZERO_CONTENTS(&remounted); - platform_default_log("cache_test: allocator recovery bootstrap test started\n"); + platform_default_log( + "cache_test: allocator recovery bootstrap test started\n"); - rc = rc_allocator_init( - &original, cfg, io, hid, platform_get_module_id()); + rc = rc_allocator_init(&original, cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { goto cleanup; } original_live = TRUE; - rc = allocator_alloc_super_addr( - (allocator *)&original, root_id, &super_addr); + rc = + allocator_alloc_super_addr((allocator *)&original, root_id, &super_addr); if (!SUCCESS(rc)) { goto cleanup; } @@ -239,8 +239,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } - rc = rc_allocator_mount( - &remounted, cfg, io, hid, platform_get_module_id()); + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { platform_error_log("cache_test: normal mount did not fall back to its " "surviving clean-state record\n"); @@ -260,8 +259,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, rc_allocator_deinit(&remounted); remounted_live = FALSE; - rc = rc_allocator_mount( - &remounted, cfg, io, hid, platform_get_module_id()); + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); if (rc.r != STATUS_INVALID_STATE.r) { platform_error_log("cache_test: normal mount trusted an allocator after " "a simulated mounted crash\n"); @@ -278,7 +276,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, recovery_live = TRUE; uint64 mounted_super_addr = 0; - rc = allocator_get_super_addr( + rc = allocator_get_super_addr( (allocator *)&recovery, root_id, &mounted_super_addr); if (!SUCCESS(rc) || mounted_super_addr != super_addr) { platform_error_log("cache_test: recovery lost a persisted superblock " @@ -294,8 +292,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, rc = STATUS_TEST_FAILED; goto cleanup; } - for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) - { + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) { uint64 extent_addr = extent_no * cfg->io_cfg->extent_size; if (allocator_get_refcount((allocator *)&recovery, extent_addr) != AL_ONE_REF) @@ -333,8 +330,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, rc_allocator_abort_recovery(&recovery); recovery_live = FALSE; - rc = rc_allocator_mount( - &remounted, cfg, io, hid, platform_get_module_id()); + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); if (rc.r != STATUS_INVALID_STATE.r) { platform_error_log("cache_test: normal mount trusted an unclean " "allocator map\n"); @@ -368,8 +364,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, rc_allocator_unmount(&recovery); recovery_live = FALSE; - rc = rc_allocator_mount( - &remounted, cfg, io, hid, platform_get_module_id()); + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { goto cleanup; } @@ -415,7 +410,7 @@ static void cache_test_writeback_fence_thread(void *arg) { cache_fence_test_context *ctxt = (cache_fence_test_context *)arg; - ctxt->status = cache_writeback_fence(ctxt->cc); + ctxt->status = cache_writeback_dirty(ctxt->cc); __atomic_store_n(&ctxt->finished, TRUE, __ATOMIC_RELEASE); } @@ -431,19 +426,18 @@ test_cache_writeback_fence(cache *cc, clockcache_config *cfg, platform_heap_id hid) { - platform_status rc = STATUS_OK; - uint64 pages_per_extent = - cache_config_pages_per_extent(&cfg->super); - uint64 *addr_arr = NULL; - page_handle *old_page = NULL; - page_handle *new_page = NULL; - bool32 new_claimed = FALSE, new_locked = FALSE; - bool32 fence_thread_started = FALSE; - const uint8 old_baseline = 0x11, old_value = 0x22; - const uint8 new_baseline = 0x33, new_value = 0x44; + platform_status rc = STATUS_OK; + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *old_page = NULL; + page_handle *new_page = NULL; + bool32 new_claimed = FALSE, new_locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 old_baseline = 0x11, old_value = 0x22; + const uint8 new_baseline = 0x33, new_value = 0x44; cache_fence_test_context fence_ctxt = { .cc = cc, .finished = FALSE, .status = STATUS_OK}; - platform_thread fence_thread; + platform_thread fence_thread; platform_default_log("cache_test: writeback fence test started\n"); platform_assert(cfg->page_capacity >= 2 * pages_per_extent); @@ -559,10 +553,8 @@ test_cache_writeback_fence(cache *cc, } /* The dirty, unlocked page was flushed by the fence. */ if (SUCCESS(rc)) { - rc = cache_test_verify_disk_page(cc, - addr_arr[0], - cache_config_page_size(&cfg->super), - old_value); + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), old_value); } /* The write-locked page was skipped, so its baseline is still on disk. */ if (SUCCESS(rc)) { @@ -604,17 +596,16 @@ test_cache_writeback_fence_claimed_page(cache *cc, clockcache_config *cfg, platform_heap_id hid) { - platform_status rc = STATUS_OK; - uint64 pages_per_extent = - cache_config_pages_per_extent(&cfg->super); - uint64 *addr_arr = NULL; - page_handle *page = NULL; - bool32 claimed = FALSE, locked = FALSE; - bool32 fence_thread_started = FALSE; - const uint8 baseline = 0x55, final_value = 0x66; + platform_status rc = STATUS_OK; + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *page = NULL; + bool32 claimed = FALSE, locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 baseline = 0x55, final_value = 0x66; cache_fence_test_context fence_ctxt = { .cc = cc, .finished = FALSE, .status = STATUS_OK}; - platform_thread fence_thread; + platform_thread fence_thread; platform_default_log("cache_test: claimed-page fence test started\n"); @@ -671,7 +662,7 @@ test_cache_writeback_fence_claimed_page(cache *cc, } platform_status join_rc = platform_thread_join(&fence_thread); - fence_thread_started = FALSE; + fence_thread_started = FALSE; if (!SUCCESS(join_rc)) { rc = join_rc; goto cleanup; @@ -688,13 +679,14 @@ test_cache_writeback_fence_claimed_page(cache *cc, goto cleanup; } - /* Drop the claim; the page is now cleanable and a second fence flushes it. */ + /* Drop the claim; the page is now cleanable and a second fence flushes it. + */ cache_unclaim(cc, page); claimed = FALSE; cache_unget(cc, page); page = NULL; - rc = cache_writeback_fence(cc); + rc = cache_writeback_dirty(cc); if (!SUCCESS(rc)) { goto cleanup; } @@ -732,8 +724,8 @@ test_cache_writeback_fence_claimed_page(cache *cc, if (addr_arr != NULL) { cache_flush(cc); - allocator *al = cache_get_allocator(cc); - refcount ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); platform_assert(ref == AL_NO_REFS); cache_extent_discard(cc, addr_arr[0], PAGE_TYPE_MISC); ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); @@ -749,6 +741,166 @@ test_cache_writeback_fence_claimed_page(cache *cc, return rc; } +typedef struct { + cache *cc; + uint64 *addrs; + uint64 num_addrs; + uint64 page_size; + volatile bool32 stop; +} cache_hammer_context; + +/* + * Continuously dirty and asynchronously write back a rotating set of pages, so + * the I/O subsystem never reaches global quiescence. + */ +static void +cache_test_hammer_thread(void *arg) +{ + cache_hammer_context *ctxt = (cache_hammer_context *)arg; + uint64 i = 0; + while (!__atomic_load_n(&ctxt->stop, __ATOMIC_ACQUIRE)) { + uint64 addr = ctxt->addrs[i % ctxt->num_addrs]; + page_handle *page = cache_get(ctxt->cc, addr, TRUE, PAGE_TYPE_MISC); + if (cache_try_claim(ctxt->cc, page)) { + cache_lock(ctxt->cc, page); + memset(page->data, (uint8)i, ctxt->page_size); + cache_unlock(ctxt->cc, page); + cache_unclaim(ctxt->cc, page); + cache_page_writeback(ctxt->cc, page, FALSE, PAGE_TYPE_MISC); + } + cache_unget(ctxt->cc, page); + i++; + } +} + +/* + * A writeback fence must drain only the pages dirty at its cut and then return, + * even while another thread keeps dirtying and writing back other pages. The + * previous io_wait_all implementation waited for *global* I/O quiescence, which + * this workload never reaches, so it would hang here. + */ +static platform_status +test_cache_writeback_fence_liveness(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 page_size = cache_config_page_size(&cfg->super); + uint64 pages_per_extent = + cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + const uint8 baseline = 0x11, target_value = 0x77; + bool32 hammer_started = FALSE, fence_started = FALSE; + cache_hammer_context hammer_ctxt = {.cc = cc, .stop = FALSE}; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread hammer_thread, fence_thread; + + platform_default_log("cache_test: writeback fence liveness test started\n"); + platform_assert(cfg->page_capacity >= 2 * pages_per_extent); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, 2 * pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 2); + if (!SUCCESS(rc)) { + platform_free(hid, addr_arr); + addr_arr = NULL; + goto cleanup; + } + rc = cache_test_fill_page(cc, addr_arr[0], page_size, baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + /* Dirty the target page (unlocked); the fence must flush it. */ + rc = cache_test_fill_page(cc, addr_arr[0], page_size, target_value); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* Hammer the second extent's pages to keep I/O perpetually in flight. */ + hammer_ctxt.addrs = &addr_arr[pages_per_extent]; + hammer_ctxt.num_addrs = pages_per_extent; + hammer_ctxt.page_size = page_size; + rc = platform_thread_create( + &hammer_thread, FALSE, cache_test_hammer_thread, &hammer_ctxt, hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + hammer_started = TRUE; + + rc = platform_thread_create( + &fence_thread, FALSE, cache_test_writeback_fence_thread, &fence_ctxt, hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_started = TRUE; + + /* The fence must return despite the ongoing hammer activity. */ + timestamp wait_start = platform_get_timestamp(); + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(10)) { + platform_error_log( + "cache_test: fence did not terminate under concurrent writes\n"); + rc = STATUS_TIMEDOUT; + break; + } + platform_sleep_ns(1000); + } + +cleanup: + /* Stop the hammer first so I/O can quiesce even if the fence is stuck. */ + if (hammer_started) { + __atomic_store_n(&hammer_ctxt.stop, TRUE, __ATOMIC_RELEASE); + platform_status jrc = platform_thread_join(&hammer_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; + } + } + if (fence_started) { + platform_status jrc = platform_thread_join(&fence_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + /* The target page, dirty at the cut, must have been made durable. */ + if (SUCCESS(rc)) { + rc = cache_durable_barrier(cc); + } + if (SUCCESS(rc)) { + rc = cache_test_verify_disk_page(cc, addr_arr[0], page_size, target_value); + } + + if (addr_arr != NULL) { + cache_flush(cc); + for (uint32 i = 0; i < 2; i++) { + uint64 addr = addr_arr[i * pages_per_extent]; + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr, PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + } + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: writeback fence liveness test passed\n"); + } else { + platform_default_log("cache_test: writeback fence liveness test failed\n"); + } + return rc; +} + platform_status test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) { @@ -767,6 +919,11 @@ test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) goto exit; } + rc = test_cache_writeback_fence_liveness(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + /* allocate twice as many pages as the cache capacity */ uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); uint32 extent_capacity = cfg->page_capacity / pages_per_extent; @@ -1644,8 +1801,8 @@ cache_test(int argc, char *argv[]) goto cleanup; } - rc = test_rc_allocator_recovery_bootstrap( - &system_cfg.allocator_cfg, io, hid); + rc = + test_rc_allocator_recovery_bootstrap(&system_cfg.allocator_cfg, io, hid); if (!SUCCESS(rc)) { goto destroy_iohandle; } diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index 6a16d05f5..fa17d8b41 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -71,19 +71,18 @@ test_log_crash(clockcache *cc, * (memtable_generation, leaf_generation) order below. */ uint64 memtable_generation = 1 - i / LOG_TEST_LEAVES_PER_MEMTABLE; - uint64 leaf_generation = LOG_TEST_LEAVES_PER_MEMTABLE - 1 - - i % LOG_TEST_LEAVES_PER_MEMTABLE; + uint64 leaf_generation = + LOG_TEST_LEAVES_PER_MEMTABLE - 1 - i % LOG_TEST_LEAVES_PER_MEMTABLE; entry_num = memtable_generation * LOG_TEST_LEAVES_PER_MEMTABLE + leaf_generation; } - key skey = - test_key(&keybuffer, - TEST_RANDOM, - entry_num, - 0, - 0, - 1 + (entry_num % key_size), - 0); + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); generate_test_message(gen, entry_num, &msg); int log_rc = log_write(logh, skey, @@ -95,7 +94,7 @@ test_log_crash(clockcache *cc, rc = log_seal(logh); platform_assert_status_ok(rc); - rc = cache_writeback_fence((cache *)cc); + rc = cache_writeback_dirty((cache *)cc); platform_assert_status_ok(rc); rc = cache_durable_barrier((cache *)cc); platform_assert_status_ok(rc); @@ -121,8 +120,7 @@ test_log_crash(clockcache *cc, uint64 leaf_generation; shard_log_iterator_curr_generations( &itor, &memtable_generation, &leaf_generation); - platform_assert( - memtable_generation == i / LOG_TEST_LEAVES_PER_MEMTABLE); + platform_assert(memtable_generation == i / LOG_TEST_LEAVES_PER_MEMTABLE); platform_assert(leaf_generation == i % LOG_TEST_LEAVES_PER_MEMTABLE); if (data_key_compare(cfg->data_cfg, skey, returned_key) || message_lex_cmp(mmessage, returned_message)) @@ -166,7 +164,7 @@ test_log_write_range(log_handle *logh, for (uint64 i = 0; i < num_entries; i++) { uint64 entry_num = first_entry + i; - key skey = test_key(&keybuffer, + key skey = test_key(&keybuffer, TEST_RANDOM, entry_num, 0, @@ -174,11 +172,8 @@ test_log_write_range(log_handle *logh, 1 + (entry_num % key_size), 0); generate_test_message(gen, entry_num, &msg); - int log_rc = log_write(logh, - skey, - merge_accumulator_to_message(&msg), - entry_num, - 0); + int log_rc = log_write( + logh, skey, merge_accumulator_to_message(&msg), entry_num, 0); platform_assert(log_rc == 0); } @@ -228,9 +223,9 @@ test_log_verify_segment(cache *cc, platform_assert(memtable_generation == entry_num); platform_assert(leaf_generation == 0); platform_assert(data_key_compare(cfg->data_cfg, skey, returned_key) == 0); - platform_assert(message_lex_cmp( - merge_accumulator_to_message(&msg), returned_message) - == 0); + platform_assert( + message_lex_cmp(merge_accumulator_to_message(&msg), returned_message) + == 0); rc = iterator_next(itorh); platform_assert_status_ok(rc); } @@ -257,8 +252,8 @@ test_log_rotate(clockcache *cc, test_message_generator *gen, uint64 key_size) { - const uint64 old_first = 1000, old_count = 16; - const uint64 new_first = 2000, new_count = 16; + const uint64 old_first = 1000, old_count = 16; + const uint64 new_first = 2000, new_count = 16; log_segment_info sealed, fresh; platform_status rc = shard_log_init(log, (cache *)cc, cfg); @@ -275,7 +270,7 @@ test_log_rotate(clockcache *cc, platform_assert(sealed.meta_addr != fresh.meta_addr); platform_assert(sealed.magic != fresh.magic); - rc = cache_writeback_fence((cache *)cc); + rc = cache_writeback_dirty((cache *)cc); platform_assert_status_ok(rc); rc = cache_durable_barrier((cache *)cc); platform_assert_status_ok(rc); @@ -284,20 +279,14 @@ test_log_rotate(clockcache *cc, rc = clockcache_init( cc, cache_cfg, io, al, "rotated-old", hid, platform_get_module_id()); platform_assert_status_ok(rc); - test_log_verify_segment((cache *)cc, - cfg, - &sealed, - gen, - hid, - key_size, - old_first, - old_count); + test_log_verify_segment( + (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); test_log_write_range( (log_handle *)log, gen, hid, key_size, new_first, new_count); rc = log_seal((log_handle *)log); platform_assert_status_ok(rc); - rc = cache_writeback_fence((cache *)cc); + rc = cache_writeback_dirty((cache *)cc); platform_assert_status_ok(rc); rc = cache_durable_barrier((cache *)cc); platform_assert_status_ok(rc); @@ -306,22 +295,10 @@ test_log_rotate(clockcache *cc, rc = clockcache_init( cc, cache_cfg, io, al, "rotated-new", hid, platform_get_module_id()); platform_assert_status_ok(rc); - test_log_verify_segment((cache *)cc, - cfg, - &sealed, - gen, - hid, - key_size, - old_first, - old_count); - test_log_verify_segment((cache *)cc, - cfg, - &fresh, - gen, - hid, - key_size, - new_first, - new_count); + test_log_verify_segment( + (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); + test_log_verify_segment( + (cache *)cc, cfg, &fresh, gen, hid, key_size, new_first, new_count); shard_log_segment_discard((cache *)cc, &sealed); shard_log_zap(log); @@ -353,9 +330,8 @@ test_log_large_message(cache *cc, merge_accumulator_set_class(&msg, MESSAGE_TYPE_INSERT); memset(merge_accumulator_data(&msg), 'L', value_len); - int log_rc = - log_write( - (log_handle *)log, skey, merge_accumulator_to_message(&msg), 0, 0); + int log_rc = log_write( + (log_handle *)log, skey, merge_accumulator_to_message(&msg), 0, 0); platform_assert(log_rc == 0); merge_accumulator filler; @@ -366,19 +342,18 @@ test_log_large_message(cache *cc, memset( merge_accumulator_data(&filler), 'f', merge_accumulator_length(&filler)); for (uint64 i = 1; i < 16; i++) { - log_rc = log_write( - (log_handle *)log, - skey, - merge_accumulator_to_message(&filler), - i / 4, - i % 4); + log_rc = log_write((log_handle *)log, + skey, + merge_accumulator_to_message(&filler), + i / 4, + i % 4); platform_assert(log_rc == 0); } merge_accumulator_deinit(&filler); rc = log_seal((log_handle *)log); platform_assert_status_ok(rc); - rc = cache_writeback_fence(cc); + rc = cache_writeback_dirty(cc); platform_assert_status_ok(rc); rc = cache_durable_barrier(cc); platform_assert_status_ok(rc); @@ -433,11 +408,8 @@ test_log_thread(void *arg) for (i = thread_id * num_entries; i < (thread_id + 1) * num_entries; i++) { key skey = test_key(&keybuf, TEST_RANDOM, i, 0, 0, key_size, 0); generate_test_message(gen, i, &msg); - int log_rc = log_write(logh, - skey, - merge_accumulator_to_message(&msg), - i / 1024, - i % 1024); + int log_rc = log_write( + logh, skey, merge_accumulator_to_message(&msg), i / 1024, i % 1024); platform_assert(log_rc == 0); } From f5ed61502fd47e96dcfaeb683ac23a3d435872e5 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 23:05:42 -0700 Subject: [PATCH 06/13] type decl cleanup Signed-off-by: Rob Johnson --- src/cache.h | 84 ++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/src/cache.h b/src/cache.h index 8e37827b2..598df0f5c 100644 --- a/src/cache.h +++ b/src/cache.h @@ -94,19 +94,6 @@ cache_config_extent_page(const cache_config *cfg, uint64 extent_addr, uint64 i) return extent_addr + i * cache_config_page_size(cfg); } -typedef void (*cache_generic_fn)(cache *cc); -typedef uint64 (*cache_generic_uint64_fn)(cache *cc); -typedef void (*page_generic_fn)(cache *cc, page_handle *page); -typedef platform_status (*cache_durable_barrier_fn)(cache *cc); -typedef platform_status (*cache_writeback_dirty_fn)(cache *cc); - -typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); -typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); -typedef page_handle *(*page_get_fn)(cache *cc, - uint64 addr, - bool32 blocking, - page_type type); - #define PAGE_GET_ASYNC_STATE_BUFFER_SIZE (2048) typedef union page_get_async_state_payload { uint8 bytes[PAGE_GET_ASYNC_STATE_BUFFER_SIZE]; @@ -120,6 +107,11 @@ typedef struct page_get_async_state_buffer { page_get_async_state_payload payload; } page_get_async_state_buffer; +typedef void (*cache_generic_void_fn)(cache *cc); +typedef uint64 (*cache_generic_uint64_fn)(cache *cc); +typedef platform_status (*cache_generic_status_fn)(cache *cc); +typedef void (*page_generic_fn)(cache *cc, page_handle *page); + typedef void (*page_get_async_state_init_fn)(void *payload, cache *cc, uint64 addr, @@ -129,6 +121,12 @@ typedef void (*page_get_async_state_init_fn)(void *payload, typedef async_status (*page_get_async_fn)(void *payload); typedef page_handle *(*page_get_async_state_result_fn)(void *payload); +typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); +typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); +typedef page_handle *(*page_get_fn)(cache *cc, + uint64 addr, + bool32 blocking, + page_type type); typedef bool32 (*page_try_claim_fn)(cache *cc, page_handle *page); typedef void (*page_writeback_fn)(cache *cc, page_handle *page, @@ -165,36 +163,36 @@ typedef struct cache_ops { page_get_async_fn page_get_async; page_get_async_state_result_fn page_get_async_result; - page_generic_fn page_unget; - page_try_claim_fn page_try_claim; - page_generic_fn page_unclaim; - page_generic_fn page_lock; - page_generic_fn page_unlock; - page_prefetch_fn page_prefetch; - page_prefetch_fn page_prefetch_page; - page_generic_fn page_pin; - page_generic_fn page_unpin; - page_writeback_fn page_writeback; - extent_writeback_fn extent_writeback; - cache_generic_fn flush; - cache_writeback_dirty_fn writeback_dirty; - cache_durable_barrier_fn durable_barrier; - evict_fn evict; - cache_generic_fn cleanup; - page_addr_pred_fn in_use; - page_addr_fn assert_ungot; - cache_generic_fn assert_free; - validate_page_fn validate_page; - cache_present_fn cache_present; - cache_print_fn print; - cache_print_fn print_stats; - io_stats_fn io_stats; - cache_generic_fn reset_stats; - count_dirty_fn count_dirty; - page_get_read_ref_fn page_get_read_ref; - enable_sync_get_fn enable_sync_get; - get_allocator_fn get_allocator; - cache_config_fn get_config; + page_generic_fn page_unget; + page_try_claim_fn page_try_claim; + page_generic_fn page_unclaim; + page_generic_fn page_lock; + page_generic_fn page_unlock; + page_prefetch_fn page_prefetch; + page_prefetch_fn page_prefetch_page; + page_generic_fn page_pin; + page_generic_fn page_unpin; + page_writeback_fn page_writeback; + extent_writeback_fn extent_writeback; + cache_generic_void_fn flush; + cache_generic_status_fn writeback_dirty; + cache_generic_status_fn durable_barrier; + evict_fn evict; + cache_generic_void_fn cleanup; + page_addr_pred_fn in_use; + page_addr_fn assert_ungot; + cache_generic_void_fn assert_free; + validate_page_fn validate_page; + cache_present_fn cache_present; + cache_print_fn print; + cache_print_fn print_stats; + io_stats_fn io_stats; + cache_generic_void_fn reset_stats; + count_dirty_fn count_dirty; + page_get_read_ref_fn page_get_read_ref; + enable_sync_get_fn enable_sync_get; + get_allocator_fn get_allocator; + cache_config_fn get_config; } cache_ops; // To sub-class cache, make a cache your first field; From 68dbb43c8414bf9140926bae784e639294ebbf79 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 23:08:14 -0700 Subject: [PATCH 07/13] clockcache.h cleanup Signed-off-by: Rob Johnson --- src/clockcache.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/clockcache.h b/src/clockcache.h index 421f070ee..cb5929fd6 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -10,7 +10,6 @@ #pragma once #include "platform_buffer.h" -#include "platform_mutex.h" #include "platform_threads.h" #include "allocator.h" #include "cache.h" @@ -74,11 +73,12 @@ struct clockcache_entry { page_handle page; volatile entry_status status; // Generation in which this page's current dirty interval began; 0 when the - // page is clean or free. A writeback fence drains every entry whose - // generation is at or below the cut it took (see clockcache_writeback_dirty). - volatile uint64 dirty_generation; - page_type type; - async_wait_queue waiters; + // page is clean or free. A writeback_dirty() drains every entry whose + // generation is at or below the cut it took (see + // clockcache_writeback_dirty). + volatile uint64 dirty_generation; + page_type type; + async_wait_queue waiters; #ifdef RECORD_ACQUISITION_STACKS int next_history_record; history_record history[NUM_HISTORY_RECORDS]; @@ -144,10 +144,9 @@ struct clockcache { volatile bool32 *batch_busy; // Convenience pointer for batch_bh uint64 cleaner_gap; - // Monotonic generation counter. A writeback fence atomically increments it + // Monotonic generation counter. A writeback_dirty() atomically increments it // to take a "cut", then drains every entry stamped with a generation at or - // below that cut. Concurrent fences are safe: each takes a distinct cut and - // waits only on its own I/O context, so no lock is needed. + // below that cut. uint64 dirty_generation; volatile struct { From a1fc2ea48c1864f373dd6efbe6a826f18bab0df4 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 23:23:48 -0700 Subject: [PATCH 08/13] cleanups Signed-off-by: Rob Johnson --- src/clockcache.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/clockcache.c b/src/clockcache.c index cfa48e346..12ed75c05 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -845,9 +845,9 @@ clockcache_ok_to_writeback(clockcache *cc, * Returns FALSE only if the page is genuinely not writeback-able (locked, * claimed, already in writeback, clean, ...). The CC_ACCESSED bit can flip * (a reader sets it, the clock hand clears it) between the two - *compare-and- swaps, so we retry as long as the status remains one of the - *cleanable states rather than spuriously failing on a page that stayed - *cleanable. + * compare-and- swaps, so we retry as long as the status remains one of the + * cleanable states rather than spuriously failing on a page that stayed + * cleanable. *---------------------------------------------------------------------- */ static inline bool32 @@ -1171,7 +1171,8 @@ clockcache_writeback_dirty(clockcache *cc) // Bulk-issue writeback for every dirty, unlocked page so the writes pipeline // rather than draining one batch at a time. for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { - platform_status result = clockcache_batch_start_writeback(cc, batch, TRUE); + platform_status result = + clockcache_batch_start_writeback(cc, batch, TRUE); if (!SUCCESS(result)) { return result; } @@ -3588,8 +3589,8 @@ clockcache_init(clockcache *cc, // OUT cc->dirty_generation = 1; /* lookup maps addrs to entries, entry contains the entries themselves */ - platform_status rc = platform_buffer_init(&cc->lookup_bh, - allocator_page_capacity * sizeof(cc->lookup[0])); + platform_status rc = platform_buffer_init( + &cc->lookup_bh, allocator_page_capacity * sizeof(cc->lookup[0])); if (!SUCCESS(rc)) { platform_error_log("clockcache_init: failed to allocate lookup table " "(%lu bytes): %s\n", From b4a055229aea0a51329d45dfba533aabfc2751d1 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 18 Jul 2026 00:01:40 -0700 Subject: [PATCH 09/13] clockcache.c assert on async io contract violation Signed-off-by: Rob Johnson --- src/clockcache.c | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/clockcache.c b/src/clockcache.c index 12ed75c05..f006d222a 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -1102,21 +1102,18 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) cc->stats[tid].writes_issued++; } - if (io_async_run(state->iostate) == ASYNC_STATUS_DONE) { - rc = io_async_state_get_result(state->iostate); - if (SUCCESS(rc)) { - platform_error_log( - "clockcache_batch_start_writeback: async write for addr " - "%lu completed without invoking its callback\n", - first_addr); - rc = STATUS_IO_ERROR; - } - clockcache_abort_writeback_range(cc, first_addr, end_addr); - io_async_state_deinit(state->iostate); - platform_free(PROCESS_PRIVATE_HEAP_ID, state); - result = rc; - goto close_log; - } + // The IO layer must run writeback asynchronously and complete it via + // clockcache_write_callback (which does the dirty->clean bookkeeping). + // A synchronous completion would skip that callback and strand the + // pages in CC_WRITEBACK, so a broken contract is a fatal correctness + // error. + async_status arc = io_async_run(state->iostate); + platform_assert( + arc == ASYNC_STATUS_RUNNING, + "clockcache_batch_start_writeback: async writeback for addr %lu " + "completed synchronously; the IO layer must complete it via the " + "write callback", + first_addr); } } From b45f857ec6be780ce603419e6131d42f0afc8342 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 18 Jul 2026 00:12:05 -0700 Subject: [PATCH 10/13] clockcache.c cleanup control flow Signed-off-by: Rob Johnson --- src/clockcache.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/clockcache.c b/src/clockcache.c index f006d222a..bd59647b7 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -1424,12 +1424,10 @@ clockcache_get_free_page(clockcache *cc, uint64 end_entry = start_entry + CC_ENTRIES_PER_BATCH; for (entry_no = start_entry; entry_no < end_entry; entry_no++) { entry = &cc->entry[entry_no]; - if (entry->status == CC_FREE_STATUS) { - bool32 reserved = __sync_bool_compare_and_swap( - &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS); - if (!reserved) { - continue; - } + if (entry->status == CC_FREE_STATUS + && __sync_bool_compare_and_swap( + &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS)) + { // A page that begins dirty (a fresh allocation) must carry a dirty // generation, just like a clean->dirty transition. The entry is From 72315327112646e52c34692402ce23c6d98f4af5 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 18 Jul 2026 00:15:39 -0700 Subject: [PATCH 11/13] formatting Signed-off-by: Rob Johnson --- src/log.h | 4 +--- src/memtable.c | 7 +++--- src/mini_allocator.c | 40 +++++++++++++++-------------------- src/mini_allocator.h | 12 +++++------ src/rc_allocator.c | 36 ++++++++++++++----------------- src/rc_allocator.h | 4 ++-- src/shard_log.c | 25 +++++++++++----------- src/shard_log.h | 5 ++--- tests/functional/btree_test.c | 28 ++++++++++++------------ tests/functional/cache_test.c | 29 ++++++++++++++----------- tests/unit/splinter_test.c | 10 ++------- 11 files changed, 92 insertions(+), 108 deletions(-) diff --git a/src/log.h b/src/log.h index 3dd27172d..30366edc3 100644 --- a/src/log.h +++ b/src/log.h @@ -98,9 +98,7 @@ log_seal(log_handle *log) } static inline platform_status -log_rotate(log_handle *log, - log_segment_info *sealed, - log_segment_info *fresh) +log_rotate(log_handle *log, log_segment_info *sealed, log_segment_info *fresh) { return log->ops->rotate(log, sealed, fresh); } diff --git a/src/memtable.c b/src/memtable.c index e8c8d95e0..e8b8969bc 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -403,8 +403,7 @@ memtable_context_init_at_generation(memtable_context *ctxt, batch_rwlock_init(&ctxt->rwlock); - for (uint64 generation_offset = 0; - generation_offset < cfg->max_memtables; + for (uint64 generation_offset = 0; generation_offset < cfg->max_memtables; generation_offset++) { uint64 generation = first_generation + generation_offset; @@ -420,8 +419,8 @@ memtable_context_init_at_generation(memtable_context *ctxt, * Otherwise, the checkpoint has incorporated every generation before * first_generation. */ - ctxt->generation_retired = first_generation == 0 ? (uint64)-1 - : first_generation - 1; + ctxt->generation_retired = + first_generation == 0 ? (uint64)-1 : first_generation - 1; ctxt->is_empty = TRUE; diff --git a/src/mini_allocator.c b/src/mini_allocator.c index 3142a6a73..143d374dd 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -925,9 +925,8 @@ mini_prefetch(cache *cc, page_type type, uint64 meta_head) static platform_status mini_recovery_corruption(const char *reason, uint64 addr) { - platform_error_log("Malformed mini allocator metadata: %s (addr=%lu).\n", - reason, - addr); + platform_error_log( + "Malformed mini allocator metadata: %s (addr=%lu).\n", reason, addr); return STATUS_INVALID_STATE; } @@ -936,8 +935,7 @@ mini_recovery_valid_geometry(const allocator_config *cfg) { if (cfg == NULL || cfg->io_cfg == NULL || cfg->capacity == 0 || cfg->io_cfg->page_size == 0 || cfg->io_cfg->extent_size == 0 - || cfg->io_cfg->page_size - < offsetof(mini_meta_hdr, entry_buffer) + || cfg->io_cfg->page_size < offsetof(mini_meta_hdr, entry_buffer) || cfg->io_cfg->extent_size < cfg->io_cfg->page_size || cfg->io_cfg->extent_size % cfg->io_cfg->page_size != 0 || cfg->capacity % cfg->io_cfg->extent_size != 0) @@ -990,8 +988,8 @@ mini_recovery_validate_meta_header(const mini_meta_hdr *hdr, meta_addr); } - uint64 expected_pos = first_entry_offset - + (uint64)hdr->num_entries * sizeof(meta_entry); + uint64 expected_pos = + first_entry_offset + (uint64)hdr->num_entries * sizeof(meta_entry); if (hdr->pos != expected_pos) { return mini_recovery_corruption("metadata entry position is invalid", meta_addr); @@ -1037,10 +1035,10 @@ mini_recovery_walk(cache *cc, uint64 meta_head, page_type meta_type, mini_recovery_visit_fn visit, - void *arg) + void *arg) { - if (cc == NULL || visit == NULL - || meta_type < PAGE_TYPE_FIRST || meta_type >= NUM_PAGE_TYPES) + if (cc == NULL || visit == NULL || meta_type < PAGE_TYPE_FIRST + || meta_type >= NUM_PAGE_TYPES) { return STATUS_BAD_PARAM; } @@ -1096,8 +1094,8 @@ mini_recovery_walk(cache *cc, return STATUS_IO_ERROR; } - mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; - platform_status rc = mini_recovery_validate_meta_header( + mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; + platform_status rc = mini_recovery_validate_meta_header( hdr, cfg->io_cfg->page_size, expected_prev, meta_addr); if (!SUCCESS(rc)) { cache_unget(cc, meta_page); @@ -1105,8 +1103,8 @@ mini_recovery_walk(cache *cc, } uint64 next_meta_addr = hdr->next_meta_addr; - rc = mini_recovery_validate_next_meta_addr( - cfg, meta_addr, next_meta_addr); + rc = + mini_recovery_validate_next_meta_addr(cfg, meta_addr, next_meta_addr); if (!SUCCESS(rc)) { cache_unget(cc, meta_page); return rc; @@ -1116,12 +1114,11 @@ mini_recovery_walk(cache *cc, for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { uint64 batch = meta_entry_batch(entry); page_type extent_type = meta_entry_type(entry); - uint64 extent_number = + uint64 extent_number = entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); - if (batch >= MINI_MAX_BATCHES - || extent_type < PAGE_TYPE_FIRST || extent_type >= NUM_PAGE_TYPES - || extent_number == 0 + if (batch >= MINI_MAX_BATCHES || extent_type < PAGE_TYPE_FIRST + || extent_type >= NUM_PAGE_TYPES || extent_number == 0 || extent_number > UINT64_MAX / cfg->io_cfg->extent_size) { cache_unget(cc, meta_page); @@ -1136,11 +1133,8 @@ mini_recovery_walk(cache *cc, extent_addr); } - rc = visit(extent_addr, - extent_type, - MINI_RECOVERY_EXTENT_DATA, - batch, - arg); + rc = visit( + extent_addr, extent_type, MINI_RECOVERY_EXTENT_DATA, batch, arg); if (!SUCCESS(rc)) { cache_unget(cc, meta_page); return rc; diff --git a/src/mini_allocator.h b/src/mini_allocator.h index f596d9864..8a6556798 100644 --- a/src/mini_allocator.h +++ b/src/mini_allocator.h @@ -148,7 +148,7 @@ typedef enum mini_recovery_extent_kind { MINI_RECOVERY_EXTENT_DATA, } mini_recovery_extent_kind; -#define MINI_RECOVERY_METADATA_BATCH ((uint64)-1) +#define MINI_RECOVERY_METADATA_BATCH ((uint64) - 1) typedef platform_status (*mini_recovery_visit_fn)( uint64 extent_addr, @@ -158,11 +158,11 @@ typedef platform_status (*mini_recovery_visit_fn)( void *arg); platform_status -mini_recovery_walk(cache *cc, - uint64 meta_head, - page_type meta_type, - mini_recovery_visit_fn visit, - void *arg); +mini_recovery_walk(cache *cc, + uint64 meta_head, + page_type meta_type, + mini_recovery_visit_fn visit, + void *arg); /* * mini_meta_cursor: a non-blocking cursor over the extent entries of a diff --git a/src/rc_allocator.c b/src/rc_allocator.c index 7a365fa2a..f11a45246 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -353,7 +353,7 @@ rc_allocator_read_clean_state(rc_allocator *al, } void *page = platform_buffer_getaddr(&buffer); - rc = io_read(al->io, + rc = io_read(al->io, page, al->cfg->io_cfg->page_size, rc_allocator_clean_state_addr(al->cfg, slot)); @@ -437,7 +437,7 @@ static platform_status rc_allocator_publish_clean_state(rc_allocator *al, bool32 clean_shutdown) { rc_allocator_clean_states states; - platform_status rc = rc_allocator_load_clean_states(al, &states); + platform_status rc = rc_allocator_load_clean_states(al, &states); if (!SUCCESS(rc)) { return rc; } @@ -453,9 +453,8 @@ rc_allocator_publish_clean_state(rc_allocator *al, bool32 clean_shutdown) ZERO_CONTENTS(&state); state.magic = RC_ALLOCATOR_CLEAN_STATE_MAGIC; state.format_version = RC_ALLOCATOR_CLEAN_STATE_VERSION; - state.sequence = states.have_newest - ? states.state[states.newest_slot].sequence + 1 - : 1; + state.sequence = + states.have_newest ? states.state[states.newest_slot].sequence + 1 : 1; state.clean_shutdown = clean_shutdown; state.checksum = rc_allocator_clean_state_checksum(&state); return rc_allocator_write_clean_state(al, target_slot, &state, TRUE); @@ -507,17 +506,14 @@ rc_allocator_recovery_initialize_refcounts(rc_allocator *al) return STATUS_BAD_PARAM; } - memset(al->ref_count, - 0, - rc_allocator_refcount_buffer_size(al->cfg)); + memset(al->ref_count, 0, rc_allocator_refcount_buffer_size(al->cfg)); /* * Extent 0 contains both the allocator meta page and every fixed table * superblock. The refcount table begins at extent 1; the two extents * after it hold alternating clean-state records. */ - for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) - { + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) { platform_assert(al->ref_count[extent_no] == AL_FREE); al->ref_count[extent_no] = AL_ONE_REF; rc_allocator_record_allocated_extent(al); @@ -530,7 +526,7 @@ rc_allocator_recovery_initialize_refcounts(rc_allocator *al) static bool32 rc_allocator_recovery_extent_is_reserved(const rc_allocator *al, - uint64 extent_no) + uint64 extent_no) { return extent_no < rc_allocator_reserved_extent_count(al->cfg); } @@ -603,8 +599,8 @@ rc_allocator_init_meta_page(rc_allocator *al) memset(al->meta_page->splinters, INVALID_ALLOCATOR_ROOT_ID, sizeof(al->meta_page->splinters)); - al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); - al->meta_page->format_magic = RC_ALLOCATOR_FORMAT_MAGIC; + al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); + al->meta_page->format_magic = RC_ALLOCATOR_FORMAT_MAGIC; al->meta_page->format_version = RC_ALLOCATOR_FORMAT_VERSION; return STATUS_OK; @@ -894,7 +890,8 @@ rc_allocator_mount_internal(rc_allocator *al, al->recovery_in_progress = TRUE; } else { // Load the ref counts from disk during a normal, clean mount. - status = io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); + status = + io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); if (!SUCCESS(status)) { goto deinit_buffer; } @@ -968,8 +965,8 @@ rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr) } while (TRUE) { - refcount old_ref = __atomic_load_n(&al->ref_count[extent_no], - __ATOMIC_RELAXED); + refcount old_ref = + __atomic_load_n(&al->ref_count[extent_no], __ATOMIC_RELAXED); if (old_ref == (refcount)-1) { platform_error_log("Allocator recovery refcount overflow for extent " "%lu.\n", @@ -977,8 +974,7 @@ rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr) return STATUS_LIMIT_EXCEEDED; } - refcount new_ref = - old_ref == AL_FREE ? AL_ONE_REF : old_ref + 1; + refcount new_ref = old_ref == AL_FREE ? AL_ONE_REF : old_ref + 1; if (!__sync_bool_compare_and_swap( &al->ref_count[extent_no], old_ref, new_ref)) { @@ -1172,7 +1168,7 @@ rc_allocator_alloc_super_addr(rc_allocator *al, // assign the first available slot and update the on disk metadata. al->meta_page->splinters[idx] = allocator_root_id; *addr = (1 + idx) * al->cfg->io_cfg->page_size; - platform_status io_status = rc_allocator_write_meta_page(al); + platform_status io_status = rc_allocator_write_meta_page(al); platform_assert_status_ok(io_status); status = STATUS_OK; break; @@ -1196,7 +1192,7 @@ rc_allocator_remove_super_addr(rc_allocator *al, */ if (al->meta_page->splinters[idx] == allocator_root_id) { al->meta_page->splinters[idx] = INVALID_ALLOCATOR_ROOT_ID; - platform_status status = rc_allocator_write_meta_page(al); + platform_status status = rc_allocator_write_meta_page(al); platform_assert_status_ok(status); platform_mutex_unlock(&al->lock); return; diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 079e565a6..28f8226c8 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -39,8 +39,8 @@ typedef struct ONDISK rc_allocator_meta_page { * that the two fixed clean-state extents after the refcount map are owned * by this allocator rather than by an older on-disk format. */ - uint64 format_magic; - uint64 format_version; + uint64 format_magic; + uint64 format_version; allocator_root_id splinters[RC_ALLOCATOR_MAX_ROOT_IDS]; checksum128 checksum; } rc_allocator_meta_page; diff --git a/src/shard_log.c b/src/shard_log.c index f90963232..1db98eddd 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -25,15 +25,16 @@ static uint64 shard_log_magic_idx = 0; -int shard_log_write(log_handle *log, - key tuple_key, - message msg, - uint64 memtable_generation, - uint64 leaf_generation); +int +shard_log_write(log_handle *log, + key tuple_key, + message msg, + uint64 memtable_generation, + uint64 leaf_generation); platform_status shard_log_seal(log_handle *log); platform_status -shard_log_rotate(log_handle *log, +shard_log_rotate(log_handle *log, log_segment_info *sealed, log_segment_info *fresh); void @@ -182,7 +183,7 @@ struct ONDISK log_entry { ondisk_tuple tuple; }; -#define INVALID_LOG_GENERATION ((uint64)-1) +#define INVALID_LOG_GENERATION ((uint64) - 1) static key log_entry_key(log_entry *le) @@ -254,7 +255,7 @@ get_new_page_for_thread(shard_log *log, { uint64 next_extent; - *page = shard_log_alloc(log, &next_extent); + *page = shard_log_alloc(log, &next_extent); if (*page == NULL) { return -1; } @@ -426,9 +427,8 @@ shard_log_seal(log_handle *logh) debug_assert(thread_data->offset >= sizeof(shard_log_hdr)); debug_assert(thread_data->offset <= shard_log_page_size(log->cfg)); - shard_log_hdr *hdr = (shard_log_hdr *)page->data; - log_entry *cursor = - (log_entry *)(page->data + thread_data->offset); + shard_log_hdr *hdr = (shard_log_hdr *)page->data; + log_entry *cursor = (log_entry *)(page->data + thread_data->offset); uint64 free_space = shard_log_page_size(log->cfg) - thread_data->offset; if (sizeof(log_entry) <= free_space) { log_entry_set_terminal(cursor); @@ -510,8 +510,7 @@ shard_log_rotate(log_handle *logh, } void -shard_log_segment_discard(cache *cc, - const log_segment_info *segment) +shard_log_segment_discard(cache *cc, const log_segment_info *segment) { if (segment->meta_addr == 0) { return; diff --git a/src/shard_log.h b/src/shard_log.h index 3e024d146..086381a72 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -47,7 +47,7 @@ typedef struct shard_log { uint64 meta_head; uint64 magic; /* Set once any log page has been allocated; survives sealing. */ - bool32 has_pages; + bool32 has_pages; } shard_log; typedef struct log_entry log_entry; @@ -92,8 +92,7 @@ shard_log_zap(shard_log *log); * descriptors will eventually own and release this reference instead. */ void -shard_log_segment_discard(cache *cc, - const log_segment_info *segment); +shard_log_segment_discard(cache *cc, const log_segment_info *segment); platform_status shard_log_iterator_init(cache *cc, diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 139db99e5..6d4e6d7ab 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -76,9 +76,9 @@ test_memtable_generation_init(cache *cc, test_btree_config *cfg, platform_heap_id hid) { - const uint64 first_generation = 17; - const uint64 max_memtables = 4; - memtable_config mt_cfg = *cfg->mt_cfg; + const uint64 first_generation = 17; + const uint64 max_memtables = 4; + memtable_config mt_cfg = *cfg->mt_cfg; memtable_context mt_ctxt; mt_cfg.max_memtables = max_memtables; @@ -116,12 +116,12 @@ test_memtable_generation_init(cache *cc, } rc = memtable_context_init_at_generation(&mt_ctxt, - hid, - cc, - &mt_cfg, - test_btree_process_noop, - NULL, - first_generation); + hid, + cc, + &mt_cfg, + test_btree_process_noop, + NULL, + first_generation); if (!SUCCESS(rc)) { return rc; } @@ -156,7 +156,8 @@ test_memtable_generation_init(cache *cc, deinit_recovery: memtable_context_deinit(&mt_ctxt); if (SUCCESS(rc)) { - platform_default_log("btree_test: memtable generation init test passed\n"); + platform_default_log( + "btree_test: memtable generation init test passed\n"); } return rc; } @@ -206,10 +207,9 @@ test_mini_recovery_walk(cache *cc) allocator *al = cache_get_allocator(cc); mini_allocator mini; test_mini_recovery_walk_state state; - uint64 meta_head = 0; + uint64 meta_head = 0; uint64 data_extent = 0; - platform_status rc = - allocator_alloc(al, &meta_head, PAGE_TYPE_MISC); + platform_status rc = allocator_alloc(al, &meta_head, PAGE_TYPE_MISC); if (!SUCCESS(rc)) { return rc; } @@ -238,7 +238,7 @@ test_mini_recovery_walk(cache *cc) ZERO_CONTENTS(&state); state.fail_data_visit = TRUE; - rc = mini_recovery_walk( + rc = mini_recovery_walk( cc, meta_head, PAGE_TYPE_MISC, test_mini_recovery_walk_visit, &state); if (!STATUS_IS_EQ(rc, STATUS_TEST_FAILED) || state.metadata_visits != 1 || state.data_visits != 1) diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index 7e4a06b07..8941fe2a6 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -784,13 +784,12 @@ test_cache_writeback_fence_liveness(cache *cc, clockcache_config *cfg, platform_heap_id hid) { - platform_status rc = STATUS_OK; - uint64 page_size = cache_config_page_size(&cfg->super); - uint64 pages_per_extent = - cache_config_pages_per_extent(&cfg->super); - uint64 *addr_arr = NULL; - const uint8 baseline = 0x11, target_value = 0x77; - bool32 hammer_started = FALSE, fence_started = FALSE; + platform_status rc = STATUS_OK; + uint64 page_size = cache_config_page_size(&cfg->super); + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + const uint8 baseline = 0x11, target_value = 0x77; + bool32 hammer_started = FALSE, fence_started = FALSE; cache_hammer_context hammer_ctxt = {.cc = cc, .stop = FALSE}; cache_fence_test_context fence_ctxt = { .cc = cc, .finished = FALSE, .status = STATUS_OK}; @@ -833,8 +832,11 @@ test_cache_writeback_fence_liveness(cache *cc, } hammer_started = TRUE; - rc = platform_thread_create( - &fence_thread, FALSE, cache_test_writeback_fence_thread, &fence_ctxt, hid); + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); if (!SUCCESS(rc)) { goto cleanup; } @@ -876,7 +878,8 @@ test_cache_writeback_fence_liveness(cache *cc, rc = cache_durable_barrier(cc); } if (SUCCESS(rc)) { - rc = cache_test_verify_disk_page(cc, addr_arr[0], page_size, target_value); + rc = + cache_test_verify_disk_page(cc, addr_arr[0], page_size, target_value); } if (addr_arr != NULL) { @@ -894,9 +897,11 @@ test_cache_writeback_fence_liveness(cache *cc, } if (SUCCESS(rc)) { - platform_default_log("cache_test: writeback fence liveness test passed\n"); + platform_default_log( + "cache_test: writeback fence liveness test passed\n"); } else { - platform_default_log("cache_test: writeback fence liveness test failed\n"); + platform_default_log( + "cache_test: writeback fence liveness test failed\n"); } return rc; } diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 9c550ec09..01fe15f6e 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -259,7 +259,7 @@ CTEST2(splinter, test_inserts) */ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) { - allocator *alp = (allocator *)&data->al; + allocator *alp = (allocator *)&data->al; allocator_root_id root_id = test_generate_allocator_root_id(); core_handle created, mounted, rejected, cleanup; platform_status rc; @@ -277,13 +277,7 @@ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); merge_accumulator msg; merge_accumulator_init(&msg, data->hid); - test_key(&keybuf, - TEST_RANDOM, - 1, - 0, - 0, - data->workload_cfg->key_size, - 0); + test_key(&keybuf, TEST_RANDOM, 1, 0, 0, data->workload_cfg->key_size, 0); generate_test_message(&data->gen, 1, &msg); rc = core_insert(&created, key_buffer_key(&keybuf), From 96b0b60e39cf6e8a8c23b78f621377f03064a471 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 18 Jul 2026 19:15:20 -0700 Subject: [PATCH 12/13] Cleaning up trunk Signed-off-by: Rob Johnson --- src/core.c | 56 +++++++++++++++++--------- src/trunk.c | 114 +++++++++++++++++++--------------------------------- src/trunk.h | 49 ++++++++++------------ 3 files changed, 98 insertions(+), 121 deletions(-) diff --git a/src/core.c b/src/core.c index 9af1975ab..d17af7bcc 100644 --- a/src/core.c +++ b/src/core.c @@ -467,12 +467,8 @@ core_destroy_checkpoint_storage(core_handle *spl) if (!records.valid[slot] || records.record[slot].root_addr == 0) { continue; } - rc = trunk_dec_ref(spl->cfg.trunk_node_cfg, - PROCESS_PRIVATE_HEAP_ID, - spl->cc, - spl->al, - spl->ts, - records.record[slot].root_addr); + trunk_snapshot old_snapshot = {.root_addr = records.record[slot].root_addr}; + rc = trunk_snapshot_release(&spl->trunk_context, &old_snapshot); if (!SUCCESS(rc)) { platform_error_log("core_destroy_checkpoint_storage: failed to " "release record %lu root %lu: %s\n", @@ -506,7 +502,7 @@ core_capture_checkpoint_cut(core_handle *spl, */ memtable_block_lookups(&spl->mt_ctxt); uint64 retired_generation = memtable_generation_retired(&spl->mt_ctxt); - platform_status rc = trunk_snapshot_acquire(&spl->trunk_context, snapshot); + platform_status rc = trunk_snapshot_create(&spl->trunk_context, snapshot); memtable_unblock_lookups(&spl->mt_ctxt); if (!SUCCESS(rc)) { return rc; @@ -558,7 +554,11 @@ core_publish_checkpoint_record(core_handle *spl, * persist newer log/data pages, but it does not seal or publish a logical * durable-log tail; tail sync is a separate operation. */ - rc = trunk_make_durable(&spl->trunk_context); + rc = cache_writeback_dirty(spl->cc); + if (!SUCCESS(rc)) { + goto release_snapshot; + } + rc = cache_durable_barrier(spl->cc); if (!SUCCESS(rc)) { goto release_snapshot; } @@ -628,15 +628,12 @@ core_publish_checkpoint_record(core_handle *spl, } if (old_root_addr != 0) { - rc = trunk_dec_ref(spl->cfg.trunk_node_cfg, - PROCESS_PRIVATE_HEAP_ID, - spl->cc, - spl->al, - spl->ts, - old_root_addr); + trunk_snapshot old_snapshot = {.root_addr = old_root_addr}; + rc = trunk_snapshot_release(&spl->trunk_context, &old_snapshot); if (!SUCCESS(rc)) { - platform_error_log("core_publish_checkpoint_record: trunk_dec_ref " - "failed for old root addr %lu: %s\n", + platform_error_log("core_publish_checkpoint_record: " + "trunk_snapshot_release failed for old root addr " + "%lu: %s\n", old_root_addr, platform_status_to_string(rc)); goto unlock_checkpoint; @@ -2335,8 +2332,13 @@ core_mkfs(core_handle *spl, } } - rc = trunk_context_init( - &spl->trunk_context, spl->cfg.trunk_node_cfg, hid, cc, al, ts, 0); + rc = trunk_context_init(&spl->trunk_context, + spl->cfg.trunk_node_cfg, + hid, + cc, + al, + ts, + (trunk_snapshot){.root_addr = 0}); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: trunk_context_init failed: %s\n", platform_status_to_string(rc)); @@ -2473,8 +2475,22 @@ core_mount(core_handle *spl, } } - rc = trunk_context_init( - &spl->trunk_context, spl->cfg.trunk_node_cfg, hid, cc, al, ts, root_addr); + trunk_snapshot root_snapshot; + rc = trunk_snapshot_create_from_addr(al, root_addr, &root_snapshot); + if (!SUCCESS(rc)) { + platform_error_log( + "core_mount: trunk_snapshot_create_from_addr failed: %s\n", + platform_status_to_string(rc)); + goto deinit_log; + } + + rc = trunk_context_init(&spl->trunk_context, + spl->cfg.trunk_node_cfg, + hid, + cc, + al, + ts, + root_snapshot); if (!SUCCESS(rc)) { platform_error_log("core_mount: trunk_context_init failed: %s\n", platform_status_to_string(rc)); diff --git a/src/trunk.c b/src/trunk.c index 8ebbc0b2b..4ea8bdb90 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -2546,20 +2546,32 @@ trunk_read_end(trunk_context *context) * to take a snapshot. */ platform_status -trunk_snapshot_acquire(trunk_context *context, trunk_snapshot *snapshot) +trunk_snapshot_create(trunk_context *context, trunk_snapshot *snapshot) { snapshot->root_addr = 0; trunk_read_begin(context); if (context->root != NULL) { snapshot->root_addr = context->root->ref.addr; - trunk_inc_ref(context->al, snapshot->root_addr); + trunk_addr_inc_ref(context->al, snapshot->root_addr); } trunk_read_end(context); return STATUS_OK; } +platform_status +trunk_snapshot_create_from_addr(allocator *al, + uint64 root_addr, + trunk_snapshot *snapshot) +{ + snapshot->root_addr = root_addr; + if (root_addr != 0) { + trunk_addr_inc_ref(al, root_addr); + } + return STATUS_OK; +} + platform_status trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot) { @@ -2567,16 +2579,23 @@ trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot) return STATUS_OK; } - platform_status rc = trunk_dec_ref(context->cfg, - context->hid, - context->cc, - context->al, - context->ts, - snapshot->root_addr); - if (SUCCESS(rc)) { - snapshot->root_addr = 0; + /* + * Route the release through a scratch context so the ordinary COW + * teardown path (trunk_context_deinit -> trunk_ondisk_node_ref_destroy) + * performs the single decrement snapshot's reference is owed. The scratch + * context only needs context's shared cfg/cc/al/ts; it does not need to + * share context's root. + */ + trunk_context scratch; + platform_status rc = trunk_context_init( + &scratch, context->cfg, context->hid, context->cc, context->al, + context->ts, *snapshot); + snapshot->root_addr = 0; // trunk_context_init consumes it regardless of rc + if (!SUCCESS(rc)) { + return rc; } - return rc; + trunk_context_deinit(&scratch); + return STATUS_OK; } platform_status @@ -6743,7 +6762,7 @@ trunk_context_init(trunk_context *context, cache *cc, allocator *al, task_system *ts, - uint64 root_addr) + trunk_snapshot snapshot) { memset(context, 0, sizeof(trunk_context)); @@ -6753,12 +6772,20 @@ trunk_context_init(trunk_context *context, context->al = al; context->ts = ts; - if (root_addr != 0) { + if (snapshot.root_addr != 0) { + // Adopt: snapshot already carries an owned reference (from + // trunk_snapshot_create or trunk_snapshot_create_from_addr), so this + // does not take a new one; it just gives that reference a home. context->root = trunk_ondisk_node_ref_create( - context, NEGATIVE_INFINITY_KEY, root_addr, FALSE); + context, NEGATIVE_INFINITY_KEY, snapshot.root_addr, TRUE /* adopt */); if (context->root == NULL) { platform_error_log("trunk_node_context_init: " "ondisk_node_ref_create failed\n"); + // The reference was never actually adopted (creation failed before + // that could happen), but the caller has already relinquished it. + // Discharge it here so this function always consumes its snapshot, + // regardless of outcome. + trunk_ondisk_node_dec_ref(context, snapshot.root_addr); return STATUS_NO_MEMORY; } } @@ -6784,34 +6811,6 @@ trunk_context_init(trunk_context *context, return STATUS_OK; } -void -trunk_inc_ref(allocator *al, uint64 root_addr) -{ - trunk_addr_inc_ref(al, root_addr); -} - -platform_status -trunk_dec_ref(const trunk_config *cfg, - platform_heap_id hid, - cache *cc, - allocator *al, - task_system *ts, - uint64 root_addr) -{ - trunk_context context; - platform_status rc = - trunk_context_init(&context, cfg, hid, cc, al, ts, root_addr); - if (!SUCCESS(rc)) { - platform_error_log("trunk_node_dec_ref: trunk_node_context_init failed: " - "%d\n", - rc.r); - return rc; - } - trunk_ondisk_node_dec_ref(&context, root_addr); - trunk_context_deinit(&context); - return STATUS_OK; -} - void trunk_context_deinit(trunk_context *context) { @@ -6828,37 +6827,6 @@ trunk_context_deinit(trunk_context *context) } } - -platform_status -trunk_context_clone(trunk_context *dst, trunk_context *src) -{ - platform_status rc; - trunk_ondisk_node_handle handle; - rc = trunk_init_root_handle(src, &handle); - if (!SUCCESS(rc)) { - platform_error_log("trunk_node_context_clone: trunk_init_root_handle " - "failed: %d\n", - rc.r); - return rc; - } - uint64 root_addr = handle.header_page->disk_addr; - - rc = trunk_context_init( - dst, src->cfg, src->hid, src->cc, src->al, src->ts, root_addr); - trunk_ondisk_node_handle_deinit(&handle); - return rc; -} - -platform_status -trunk_make_durable(trunk_context *context) -{ - platform_status rc = cache_writeback_dirty(context->cc); - if (!SUCCESS(rc)) { - return rc; - } - return cache_durable_barrier(context->cc); -} - /************************************ * Stats ************************************/ diff --git a/src/trunk.h b/src/trunk.h index a48dbab07..70786bec3 100644 --- a/src/trunk.h +++ b/src/trunk.h @@ -205,6 +205,14 @@ trunk_config_init(trunk_config *config, uint64 prefetch_budget, bool32 use_stats); +/* + * Initializes context with a root of snapshot. Consumes snapshot's owned + * reference regardless of outcome: on success the reference now lives in + * context (released by a later trunk_context_deinit); on failure it has + * already been discharged. Either way, the caller must not use or release + * snapshot again. A null snapshot (root_addr == 0) starts an empty trunk with + * no root, as for a freshly formatted table. + */ platform_status trunk_context_init(trunk_context *context, const trunk_config *cfg, @@ -212,45 +220,35 @@ trunk_context_init(trunk_context *context, cache *cc, allocator *al, task_system *ts, - uint64 root_addr); - -void -trunk_inc_ref(allocator *al, uint64 root_addr); - -platform_status -trunk_dec_ref(const trunk_config *cfg, - platform_heap_id hid, - cache *cc, - allocator *al, - task_system *ts, - uint64 root_addr); + trunk_snapshot snapshot); void trunk_context_deinit(trunk_context *context); -/* Create a writable snapshot of a trunk */ +/* Capture an owned reference to the current COW root without reading it. */ platform_status -trunk_context_clone(trunk_context *dst, trunk_context *src); +trunk_snapshot_create(trunk_context *context, trunk_snapshot *snapshot); -/* Capture an owned reference to the current COW root without reading it. */ +/* + * Capture an owned reference to a root_addr that is not (or not yet) any live + * context's root -- e.g. one just read out of a durable checkpoint record. + * root_addr must already be referenced by the caller (its persisted refcount + * accounts for it); this takes out an additional, independent reference. A + * null root_addr (0) produces a null snapshot without touching the allocator. + */ platform_status -trunk_snapshot_acquire(trunk_context *context, trunk_snapshot *snapshot); +trunk_snapshot_create_from_addr(allocator *al, + uint64 root_addr, + trunk_snapshot *snapshot); /* Drop an owned snapshot reference that was not published. */ platform_status trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot); -/* Make a trunk durable */ -platform_status -trunk_make_durable(trunk_context *context); - /******************************** * Mutations ********************************/ -void -trunk_modification_begin(trunk_context *context); - // Build a new trunk with the branch incorporated. The new trunk is not yet // visible to queriers. platform_status @@ -276,8 +274,6 @@ trunk_optimize(trunk_context *context, bool32 full_leaf_compactions, struct splinterdb_notification *notification); -void -trunk_modification_end(trunk_context *context); /******************************** * Queries @@ -287,9 +283,6 @@ platform_status trunk_init_root_handle(trunk_context *context, trunk_ondisk_node_handle *handle); -uint64 -trunk_ondisk_node_handle_addr(const trunk_ondisk_node_handle *handle); - void trunk_ondisk_node_handle_deinit(trunk_ondisk_node_handle *handle); From 7b8b84431f0294859d94e772be2fb2cb6ea3df6a Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sun, 19 Jul 2026 00:08:47 -0700 Subject: [PATCH 13/13] clean up mini allocator recovery Signed-off-by: Rob Johnson --- src/allocator.h | 23 ++- src/mini_allocator.c | 310 +++++++++++++++++++++------------- src/mini_allocator.h | 56 ++---- src/rc_allocator.c | 21 ++- src/rc_allocator.h | 9 +- tests/functional/btree_test.c | 104 ------------ tests/functional/cache_test.c | 188 ++++++++++++++++++++- 7 files changed, 439 insertions(+), 272 deletions(-) diff --git a/src/allocator.h b/src/allocator.h index eb83b6313..54c8b939e 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -136,6 +136,16 @@ typedef platform_status (*alloc_fn)(allocator *al, typedef refcount (*dec_ref_fn)(allocator *al, uint64 addr, page_type type); typedef refcount (*generic_ref_fn)(allocator *al, uint64 addr); +/* + * Record one logical reference to addr while rebuilding a recovery map (see + * rc_allocator_mount_recovery()). The first reference to a given extent + * establishes its nonzero allocation floor; later references increment it + * normally. addr must be the base address of a non-reserved extent. + */ +typedef platform_status (*recovery_record_reference_fn)(allocator *al, + uint64 addr, + page_type type); + typedef platform_status (*get_super_addr_fn)(allocator *al, allocator_root_id spl_id, uint64 *addr); @@ -157,9 +167,10 @@ typedef struct allocator_ops { allocator_get_config_fn get_config; alloc_fn alloc; - generic_ref_fn inc_ref; - dec_ref_fn dec_ref; - generic_ref_fn get_ref; + generic_ref_fn inc_ref; + dec_ref_fn dec_ref; + generic_ref_fn get_ref; + recovery_record_reference_fn recovery_record_reference; alloc_super_addr_fn alloc_super_addr; get_super_addr_fn get_super_addr; @@ -211,6 +222,12 @@ allocator_get_refcount(allocator *al, uint64 addr) return al->ops->get_ref(al, addr); } +static inline platform_status +allocator_recovery_record_reference(allocator *al, uint64 addr, page_type type) +{ + return al->ops->recovery_record_reference(al, addr, type); +} + static inline platform_status allocator_get_super_addr(allocator *al, allocator_root_id spl_id, uint64 *addr) { diff --git a/src/mini_allocator.c b/src/mini_allocator.c index 143d374dd..6832604ea 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -760,35 +760,78 @@ mini_deinit(cache *cc, uint64 meta_head, page_type type) *----------------------------------------------------------------------------- * mini_for_each_meta_page -- * - * Calls func on each meta_page in the mini_allocator. + * Walks the meta_page chain of the mini_allocator, calling func on each + * meta_page. + * + * If before is non-NULL, it is called once per distinct meta extent -- + * before that extent's first page is cache_get()'d -- with just the + * extent's address. This lets a caller that does not yet have allocator + * ownership of these pages (e.g. crash recovery, rebuilding the refcount + * table from scratch) establish it first, before cache_get()'s debug + * allocation checks run. Ordinary callers, which already own everything + * they are walking, pass NULL. + * + * func and before may fail; the walk stops and returns the first failure. * * Results: - * None + * STATUS_OK, the first failure from before/func, or STATUS_IO_ERROR if a + * meta page could not be read. * * Side effects: - * func may store output in arg. + * func/before may store output in arg. *----------------------------------------------------------------------------- */ -typedef void (*mini_for_each_meta_page_fn)(cache *cc, - page_type type, - page_handle *meta_page, - void *arg); +typedef platform_status (*mini_meta_extent_fn)(cache *cc, + page_type type, + uint64 extent_addr, + void *arg); -static void +typedef platform_status (*mini_for_each_meta_page_fn)(cache *cc, + page_type type, + page_handle *meta_page, + void *arg); + +static platform_status mini_for_each_meta_page(cache *cc, uint64 meta_head, page_type type, + mini_meta_extent_fn before, mini_for_each_meta_page_fn func, void *arg) { - uint64 meta_addr = meta_head; + uint64 meta_addr = meta_head; + uint64 extent_size = cache_extent_size(cc); + uint64 prior_extent = (uint64)-1; + while (meta_addr != 0) { + if (before != NULL) { + uint64 extent_addr = meta_addr - meta_addr % extent_size; + if (extent_addr != prior_extent) { + platform_status rc = before(cc, type, extent_addr, arg); + if (!SUCCESS(rc)) { + return rc; + } + prior_extent = extent_addr; + } + } + page_handle *meta_page = cache_get(cc, meta_addr, TRUE, type); - func(cc, type, meta_page, arg); - meta_addr = mini_get_next_meta_addr(meta_page); + if (meta_page == NULL) { + return STATUS_IO_ERROR; + } + + platform_status rc = + func == NULL ? STATUS_OK : func(cc, type, meta_page, arg); + uint64 next_meta_addr = + SUCCESS(rc) ? mini_get_next_meta_addr(meta_page) : 0; cache_unget(cc, meta_page); + if (!SUCCESS(rc)) { + return rc; + } + meta_addr = next_meta_addr; } + return STATUS_OK; } /* mini_for_each(): call a function on each allocated extent in the @@ -804,7 +847,7 @@ typedef struct for_each_func { void *arg; } for_each_func; -static void +static platform_status mini_for_each_meta_page_func(cache *cc, page_type type, page_handle *meta_page, @@ -821,6 +864,7 @@ mini_for_each_meta_page_func(cache *cc, fef->arg); entry = next_entry(entry); } + return STATUS_OK; } static void @@ -832,7 +876,7 @@ mini_for_each(cache *cc, { for_each_func fef = {func, out}; mini_for_each_meta_page( - cc, meta_head, type, mini_for_each_meta_page_func, &fef); + cc, meta_head, type, NULL, mini_for_each_meta_page_func, &fef); } @@ -918,7 +962,7 @@ mini_prefetch(cache *cc, page_type type, uint64 meta_head) /* * ----------------------------------------------------------------------------- - * mini_recovery_walk -- Read-only mini-allocator extent enumeration. + * mini_recover_allocations -- crash-recovery allocator-reference rebuild. * ----------------------------------------------------------------------------- */ @@ -964,12 +1008,6 @@ mini_recovery_valid_extent_addr(const allocator_config *cfg, uint64 addr) && addr <= cfg->capacity - extent_size; } -static uint64 -mini_recovery_extent_base_addr(const allocator_config *cfg, uint64 addr) -{ - return addr - addr % cfg->io_cfg->extent_size; -} - static platform_status mini_recovery_validate_meta_header(const mini_meta_hdr *hdr, uint64 page_size, @@ -1030,125 +1068,160 @@ mini_recovery_validate_next_meta_addr(const allocator_config *cfg, return STATUS_OK; } -platform_status -mini_recovery_walk(cache *cc, - uint64 meta_head, - page_type meta_type, - mini_recovery_visit_fn visit, - void *arg) +typedef struct mini_recover_state { + allocator *al; + allocator_config *cfg; + uint64 expected_prev; + uint64 page_count; + uint64 max_meta_pages; +} mini_recover_state; + +/* + * The "before" hook mini_for_each_meta_page() calls once per distinct meta + * extent, before that extent's first page is cache_get()'d. Recording the + * reference here -- ahead of the read -- is what lets a rebuilding allocator + * (which does not yet consider this extent allocated) satisfy cache_get()'s + * debug allocation check. + */ +static platform_status +mini_recover_visit_meta_extent(cache *cc, + page_type type, + uint64 extent_addr, + void *arg) { - if (cc == NULL || visit == NULL || meta_type < PAGE_TYPE_FIRST - || meta_type >= NUM_PAGE_TYPES) - { - return STATUS_BAD_PARAM; + mini_recover_state *state = (mini_recover_state *)arg; + + if (!mini_recovery_valid_extent_addr(state->cfg, extent_addr)) { + return mini_recovery_corruption("metadata extent is out of range", + extent_addr); } + return allocator_recovery_record_reference(state->al, extent_addr, type); +} - allocator *al = cache_get_allocator(cc); - if (al == NULL) { - return STATUS_BAD_PARAM; +/* + * The "func" hook mini_for_each_meta_page() calls once per meta page, with the + * page held. Validates the page and every data-extent entry on it before + * trusting any of it, and records a reference for each data extent found (its + * type is self-describing on disk, unlike a meta page's). + */ +static platform_status +mini_recover_visit_meta_page(cache *cc, + page_type type, + page_handle *meta_page, + void *arg) +{ + mini_recover_state *state = (mini_recover_state *)arg; + uint64 meta_addr = meta_page->disk_addr; + + if (state->page_count == state->max_meta_pages) { + return mini_recovery_corruption("metadata chain exceeds disk pages", + meta_addr); } - allocator_config *cfg = allocator_get_config(al); - if (!mini_recovery_valid_geometry(cfg)) { - return STATUS_BAD_PARAM; + state->page_count++; + + mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; + platform_status rc = mini_recovery_validate_meta_header( + hdr, state->cfg->io_cfg->page_size, state->expected_prev, meta_addr); + if (!SUCCESS(rc)) { + return rc; } - if (!mini_recovery_valid_page_addr(cfg, meta_head)) { - return mini_recovery_corruption("metadata head is not a page", meta_head); + + rc = mini_recovery_validate_next_meta_addr( + state->cfg, meta_addr, hdr->next_meta_addr); + if (!SUCCESS(rc)) { + return rc; } - uint64 meta_addr = meta_head; - uint64 expected_prev = 0; - uint64 prior_meta_extent = (uint64)-1; - uint64 max_meta_pages = cfg->capacity / cfg->io_cfg->page_size; + meta_entry *entry = first_entry(meta_page); + for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { + uint64 batch = meta_entry_batch(entry); + page_type extent_type = meta_entry_type(entry); + uint64 extent_number = + entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); - for (uint64 page_count = 0; meta_addr != 0; page_count++) { - if (page_count == max_meta_pages) { - return mini_recovery_corruption("metadata chain exceeds disk pages", + if (batch >= MINI_MAX_BATCHES || extent_type < PAGE_TYPE_FIRST + || extent_type >= NUM_PAGE_TYPES || extent_number == 0 + || extent_number > UINT64_MAX / state->cfg->io_cfg->extent_size) + { + return mini_recovery_corruption("metadata extent entry is invalid", meta_addr); } - /* - * Report a metadata extent before reading it. The normal mini metadata - * layout is contiguous within an extent, so this is once per physical - * metadata extent; next-link and reciprocal-prev validation below reject - * a malformed chain before it can complete a loop. - */ - uint64 meta_extent = mini_recovery_extent_base_addr(cfg, meta_addr); - if (!mini_recovery_valid_extent_addr(cfg, meta_extent)) { + uint64 extent_addr = extent_number * state->cfg->io_cfg->extent_size; + if (!mini_recovery_valid_extent_addr(state->cfg, extent_addr)) { return mini_recovery_corruption("metadata extent is out of range", - meta_extent); - } - if (meta_extent != prior_meta_extent) { - platform_status rc = visit(meta_extent, - meta_type, - MINI_RECOVERY_EXTENT_METADATA, - MINI_RECOVERY_METADATA_BATCH, - arg); - if (!SUCCESS(rc)) { - return rc; - } - prior_meta_extent = meta_extent; - } - - page_handle *meta_page = cache_get(cc, meta_addr, TRUE, meta_type); - if (meta_page == NULL) { - return STATUS_IO_ERROR; - } - - mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; - platform_status rc = mini_recovery_validate_meta_header( - hdr, cfg->io_cfg->page_size, expected_prev, meta_addr); - if (!SUCCESS(rc)) { - cache_unget(cc, meta_page); - return rc; + extent_addr); } - uint64 next_meta_addr = hdr->next_meta_addr; - rc = - mini_recovery_validate_next_meta_addr(cfg, meta_addr, next_meta_addr); + rc = allocator_recovery_record_reference( + state->al, extent_addr, extent_type); if (!SUCCESS(rc)) { - cache_unget(cc, meta_page); return rc; } - meta_entry *entry = first_entry(meta_page); - for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { - uint64 batch = meta_entry_batch(entry); - page_type extent_type = meta_entry_type(entry); - uint64 extent_number = - entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); - - if (batch >= MINI_MAX_BATCHES || extent_type < PAGE_TYPE_FIRST - || extent_type >= NUM_PAGE_TYPES || extent_number == 0 - || extent_number > UINT64_MAX / cfg->io_cfg->extent_size) - { - cache_unget(cc, meta_page); - return mini_recovery_corruption("metadata extent entry is invalid", - meta_addr); - } - - uint64 extent_addr = extent_number * cfg->io_cfg->extent_size; - if (!mini_recovery_valid_extent_addr(cfg, extent_addr)) { - cache_unget(cc, meta_page); - return mini_recovery_corruption("metadata extent is out of range", - extent_addr); - } + entry = next_entry(entry); + } - rc = visit( - extent_addr, extent_type, MINI_RECOVERY_EXTENT_DATA, batch, arg); - if (!SUCCESS(rc)) { - cache_unget(cc, meta_page); - return rc; - } + state->expected_prev = meta_addr; + return STATUS_OK; +} - entry = next_entry(entry); - } +/* + * mini_recover_allocations -- + * + * Rebuild allocator references for a finalized mini allocator without + * trusting allocator refcounts -- the discovery primitive crash recovery + * uses to reconstruct them. Records one reference for every metadata + * extent and every data-extent entry found in the on-disk mini metadata + * stream (data entries are not deduplicated: repeated entries mean + * repeated references, and are recorded exactly as often as they occur). + * + * meta_type is the page type of meta_head's own chain; each data extent's + * type is read from its own metadata entry. + * + * Validates the metadata-page chain, page-header bounds, page types, + * batches, and extent addresses before using them. A malformed on-disk + * stream returns STATUS_INVALID_STATE. This function performs no writes + * and does not use allocator refcounts to decide what to traverse. + * + * This is a physical enumeration only. Recovering logical reference + * multiplicity, and deduplicating references shared by distinct mini + * allocator roots, remains the responsibility of the higher-level + * trunk/log recovery walker. + */ +platform_status +mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type) +{ + if (cc == NULL || meta_type < PAGE_TYPE_FIRST || meta_type >= NUM_PAGE_TYPES) + { + return STATUS_BAD_PARAM; + } - cache_unget(cc, meta_page); - expected_prev = meta_addr; - meta_addr = next_meta_addr; + allocator *al = cache_get_allocator(cc); + if (al == NULL) { + return STATUS_BAD_PARAM; + } + allocator_config *cfg = allocator_get_config(al); + if (!mini_recovery_valid_geometry(cfg)) { + return STATUS_BAD_PARAM; + } + if (!mini_recovery_valid_page_addr(cfg, meta_head)) { + return mini_recovery_corruption("metadata head is not a page", meta_head); } - return STATUS_OK; + mini_recover_state state = {.al = al, + .cfg = cfg, + .expected_prev = 0, + .page_count = 0, + .max_meta_pages = cfg->capacity + / cfg->io_cfg->page_size}; + + return mini_for_each_meta_page(cc, + meta_head, + meta_type, + mini_recover_visit_meta_extent, + mini_recover_visit_meta_page, + &state); } /* @@ -1314,7 +1387,7 @@ space_use_add_extent(cache *cc, page_type type, uint64 extent_addr, void *out) *sum += cache_extent_size(cc); } -static void +static platform_status space_use_add_meta_page(cache *cc, page_type type, page_handle *meta_page, @@ -1322,6 +1395,7 @@ space_use_add_meta_page(cache *cc, { uint64 *sum = (uint64 *)out; *sum += cache_page_size(cc); + return STATUS_OK; } uint64 @@ -1330,7 +1404,7 @@ mini_space_use_bytes(cache *cc, uint64 meta_head, page_type type) uint64 total = 0; mini_for_each(cc, meta_head, type, space_use_add_extent, &total); mini_for_each_meta_page( - cc, meta_head, type, space_use_add_meta_page, &total); + cc, meta_head, type, NULL, space_use_add_meta_page, &total); return total; } diff --git a/src/mini_allocator.h b/src/mini_allocator.h index 8a6556798..5407530f9 100644 --- a/src/mini_allocator.h +++ b/src/mini_allocator.h @@ -114,55 +114,31 @@ void mini_prefetch(cache *cc, page_type type, uint64 meta_head); /* - * mini_recovery_walk -- + * mini_recover_allocations -- * - * Enumerate the physical extents reachable from a finalized mini - * allocator without looking at allocator refcounts. This is the - * discovery primitive used by crash recovery to rebuild those refcounts. + * Rebuild allocator references for a finalized mini allocator without + * trusting allocator refcounts. This is the discovery primitive crash + * recovery uses to reconstruct them: it records one allocator reference + * (via allocator_recovery_record_reference()) for every metadata extent + * and for every data-extent entry in the on-disk mini metadata stream. + * Data entries are deliberately not deduplicated: repeated entries + * represent repeated references in the mini allocator and are recorded + * exactly as often as they occur. meta_type is the page type of + * meta_head's own chain; each data extent's type is read from its own + * metadata entry, so it is not a parameter here. * - * The callback is invoked once for each metadata extent and once for - * every data-extent entry in the on-disk mini metadata stream. Data - * entries are deliberately not deduplicated: repeated entries represent - * repeated references in the mini allocator and are reported exactly as - * recorded. Metadata entries have batch - * MINI_RECOVERY_METADATA_BATCH. - * - * The walker validates the metadata-page chain, page-header bounds, page - * types, batches, and extent addresses before using them. A malformed - * on-disk stream returns STATUS_INVALID_STATE. Callback failures are - * returned unchanged. The walker performs no writes and does not use - * allocator refcounts to decide what to traverse. - * - * The callback is called for a metadata extent before the walker reads a - * page from that extent. That ordering lets an allocator-recovery caller - * establish temporary ownership before cache_get() performs its debug - * allocation checks. + * Validates the metadata-page chain, page-header bounds, page types, + * batches, and extent addresses before using them. A malformed on-disk + * stream returns STATUS_INVALID_STATE. This function performs no writes + * and does not use allocator refcounts to decide what to traverse. * * This is a physical enumeration only. Recovering logical reference * multiplicity, and deduplicating references shared by distinct mini * allocator roots, remains the responsibility of the higher-level * trunk/log recovery walker. */ -typedef enum mini_recovery_extent_kind { - MINI_RECOVERY_EXTENT_METADATA, - MINI_RECOVERY_EXTENT_DATA, -} mini_recovery_extent_kind; - -#define MINI_RECOVERY_METADATA_BATCH ((uint64) - 1) - -typedef platform_status (*mini_recovery_visit_fn)( - uint64 extent_addr, - page_type type, - mini_recovery_extent_kind kind, - uint64 batch, - void *arg); - platform_status -mini_recovery_walk(cache *cc, - uint64 meta_head, - page_type meta_type, - mini_recovery_visit_fn visit, - void *arg); +mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type); /* * mini_meta_cursor: a non-blocking cursor over the extent entries of a diff --git a/src/rc_allocator.c b/src/rc_allocator.c index f11a45246..e86d85195 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -123,6 +123,20 @@ rc_allocator_get_ref_virtual(allocator *a, uint64 addr) return rc_allocator_get_ref(al, addr); } +platform_status +rc_allocator_rebuild_acquire_extent(rc_allocator *al, + uint64 extent_addr, + page_type type); + +platform_status +rc_allocator_recovery_record_reference_virtual(allocator *a, + uint64 addr, + page_type type) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_rebuild_acquire_extent(al, addr, type); +} + platform_status rc_allocator_get_super_addr(rc_allocator *al, allocator_root_id spl_id, @@ -217,6 +231,7 @@ const static allocator_ops rc_allocator_ops = { .inc_ref = rc_allocator_inc_ref_virtual, .dec_ref = rc_allocator_dec_ref_virtual, .get_ref = rc_allocator_get_ref_virtual, + .recovery_record_reference = rc_allocator_recovery_record_reference_virtual, .get_super_addr = rc_allocator_get_super_addr_virtual, .alloc_super_addr = rc_allocator_alloc_super_addr_virtual, .remove_super_addr = rc_allocator_remove_super_addr_virtual, @@ -940,7 +955,9 @@ rc_allocator_mount_recovery(rc_allocator *al, } platform_status -rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr) +rc_allocator_rebuild_acquire_extent(rc_allocator *al, + uint64 extent_addr, + page_type type) { if (!al->recovery_in_progress) { platform_error_log("Cannot acquire allocator extent while recovery is " @@ -982,7 +999,9 @@ rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr) } if (old_ref == AL_FREE) { + platform_assert(type != PAGE_TYPE_INVALID); rc_allocator_record_allocated_extent(al); + __sync_add_and_fetch(&al->stats.extent_allocs[type], 1); } return STATUS_OK; } diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 28f8226c8..b7ada5cc5 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -136,11 +136,14 @@ rc_allocator_mount_recovery(rc_allocator *al, /* * Add one logical ownership reference for an extent while rebuilding a * recovery map. The first reference establishes the allocator's nonzero - * allocation floor (AL_ONE_REF); later references increment it normally. - * extent_addr must be the base address of a non-reserved allocator extent. + * allocation floor (AL_ONE_REF) and records it in stats.extent_allocs[type]; + * later references increment the refcount normally. extent_addr must be the + * base address of a non-reserved allocator extent. */ platform_status -rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr); +rc_allocator_rebuild_acquire_extent(rc_allocator *al, + uint64 extent_addr, + page_type type); /* * Complete a successful rebuild without performing I/O. A later normal diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 6d4e6d7ab..560e4c246 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -162,107 +162,6 @@ test_memtable_generation_init(cache *cc, return rc; } -typedef struct test_mini_recovery_walk_state { - uint64 metadata_visits; - uint64 data_visits; - uint64 metadata_extent; - uint64 data_extent; - bool32 fail_data_visit; -} test_mini_recovery_walk_state; - -static platform_status -test_mini_recovery_walk_visit(uint64 extent_addr, - page_type type, - mini_recovery_extent_kind kind, - uint64 batch, - void *arg) -{ - test_mini_recovery_walk_state *state = arg; - - if (type != PAGE_TYPE_MISC) { - return STATUS_TEST_FAILED; - } - - if (kind == MINI_RECOVERY_EXTENT_METADATA) { - if (batch != MINI_RECOVERY_METADATA_BATCH) { - return STATUS_TEST_FAILED; - } - state->metadata_visits++; - state->metadata_extent = extent_addr; - return STATUS_OK; - } - - if (kind != MINI_RECOVERY_EXTENT_DATA || batch != 0) { - return STATUS_TEST_FAILED; - } - - state->data_visits++; - state->data_extent = extent_addr; - return state->fail_data_visit ? STATUS_TEST_FAILED : STATUS_OK; -} - -static platform_status -test_mini_recovery_walk(cache *cc) -{ - allocator *al = cache_get_allocator(cc); - mini_allocator mini; - test_mini_recovery_walk_state state; - uint64 meta_head = 0; - uint64 data_extent = 0; - platform_status rc = allocator_alloc(al, &meta_head, PAGE_TYPE_MISC); - if (!SUCCESS(rc)) { - return rc; - } - - mini_init(&mini, cc, meta_head, 0, 1, PAGE_TYPE_MISC); - data_extent = mini_alloc_extent(&mini, 0, NULL); - if (data_extent == 0) { - rc = STATUS_NO_SPACE; - goto cleanup; - } - - ZERO_CONTENTS(&state); - rc = mini_recovery_walk( - cc, meta_head, PAGE_TYPE_MISC, test_mini_recovery_walk_visit, &state); - if (!SUCCESS(rc)) { - goto cleanup; - } - if (state.metadata_visits != 1 || state.data_visits != 1 - || state.metadata_extent != meta_head - || state.data_extent != data_extent) - { - platform_error_log("mini recovery walker returned unexpected extents\n"); - rc = STATUS_TEST_FAILED; - goto cleanup; - } - - ZERO_CONTENTS(&state); - state.fail_data_visit = TRUE; - rc = mini_recovery_walk( - cc, meta_head, PAGE_TYPE_MISC, test_mini_recovery_walk_visit, &state); - if (!STATUS_IS_EQ(rc, STATUS_TEST_FAILED) || state.metadata_visits != 1 - || state.data_visits != 1) - { - platform_error_log("mini recovery walker did not propagate callback " - "failure\n"); - rc = STATUS_TEST_FAILED; - goto cleanup; - } - rc = STATUS_OK; - -cleanup: - mini_release(&mini); - refcount ref = mini_dec_ref(cc, meta_head, PAGE_TYPE_MISC); - if (ref != 0 && SUCCESS(rc)) { - platform_error_log("mini recovery walker left an unexpected mini ref\n"); - rc = STATUS_TEST_FAILED; - } - if (SUCCESS(rc)) { - platform_default_log("btree_test: mini recovery walker test passed\n"); - } - return rc; -} - test_memtable_context * test_memtable_context_create(cache *cc, test_btree_config *cfg, @@ -2454,9 +2353,6 @@ btree_test(int argc, char *argv[]) rc = test_memtable_generation_init(ccp, &test_cfg, hid); platform_assert_status_ok(rc); - rc = test_mini_recovery_walk(ccp); - platform_assert_status_ok(rc); - uint64 max_tuples_per_memtable = test_cfg.mt_cfg->max_extents_per_memtable * cache_config_extent_size((cache_config *)&system_cfg.cache_cfg) / 3 diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index 8941fe2a6..6dd55db33 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -12,6 +12,7 @@ #include "rc_allocator.h" #include "cache.h" #include "clockcache.h" +#include "mini_allocator.h" #include "task.h" #include "random.h" #include "test_common.h" @@ -312,7 +313,8 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } - rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + rc = rc_allocator_rebuild_acquire_extent( + &recovery, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc) || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) != AL_ONE_REF) @@ -346,11 +348,13 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, } recovery_live = TRUE; - rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + rc = rc_allocator_rebuild_acquire_extent( + &recovery, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc)) { goto cleanup; } - rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + rc = rc_allocator_rebuild_acquire_extent( + &recovery, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc) || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) != AL_ONE_REF + 1) @@ -400,6 +404,178 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, return rc; } +#define TEST_MINI_RECOVER_NUM_DATA_EXTENTS 3 + +/* + * Build a real mini_allocator with one metadata extent and several data + * extents, durably persist it, then simulate a crash: mount a fresh allocator + * in recovery mode (which deliberately ignores the persisted refcount table) + * and confirm mini_recover_allocations() rediscovers every extent from the + * on-disk metadata alone -- including the ordering property this whole + * mechanism exists for, since a rebuilding allocator considers all of these + * extents unallocated until mini_recover_allocations() itself establishes + * their references, ahead of the cache_get() calls it issues to read them. + */ +static platform_status +test_mini_recover_allocations(allocator_config *allocator_cfg, + clockcache_config *cache_cfg, + io_handle *io, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + rc_allocator original, recovery; + clockcache original_cc, recovery_cc; + bool32 original_al_live = FALSE, original_cc_live = FALSE; + bool32 recovery_al_live = FALSE, recovery_cc_live = FALSE; + mini_allocator mini; + bool32 mini_live = FALSE; + uint64 meta_head = 0; + uint64 data_extents[TEST_MINI_RECOVER_NUM_DATA_EXTENTS] = {0}; + + platform_default_log( + "cache_test: mini_recover_allocations test started\n"); + + rc = rc_allocator_init( + &original, allocator_cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + original_al_live = TRUE; + + rc = clockcache_init(&original_cc, + cache_cfg, + io, + (allocator *)&original, + "test", + hid, + platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + original_cc_live = TRUE; + cache *original_ccp = (cache *)&original_cc; + + rc = allocator_alloc((allocator *)&original, &meta_head, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + goto cleanup; + } + mini_init(&mini, original_ccp, meta_head, 0, 1, PAGE_TYPE_MISC); + mini_live = TRUE; + + for (uint64 i = 0; i < TEST_MINI_RECOVER_NUM_DATA_EXTENTS; i++) { + data_extents[i] = mini_alloc_extent(&mini, 0, NULL); + if (data_extents[i] == 0) { + rc = STATUS_NO_SPACE; + goto cleanup; + } + } + mini_release(&mini); + mini_live = FALSE; + + /* Durably persist before "crashing": recovery must see real disk state. */ + cache_flush(original_ccp); + clockcache_deinit(&original_cc); + original_cc_live = FALSE; + rc_allocator_unmount(&original); + original_al_live = FALSE; + + rc = rc_allocator_mount_recovery( + &recovery, allocator_cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_al_live = TRUE; + + /* mount_recovery deliberately ignores the persisted map: confirm that. */ + if (allocator_get_refcount((allocator *)&recovery, meta_head) != AL_FREE) { + platform_error_log( + "cache_test: recovery mount trusted a persisted refcount\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + rc = clockcache_init(&recovery_cc, + cache_cfg, + io, + (allocator *)&recovery, + "test", + hid, + platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_cc_live = TRUE; + + /* + * Every extent below is still AL_FREE from recovery's point of view: this + * call must establish each one's reference itself, before its own + * cache_get() calls, or those cache_get()s would trip the cache's debug + * allocation check. + */ + rc = mini_recover_allocations( + (cache *)&recovery_cc, meta_head, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + platform_error_log("cache_test: mini_recover_allocations failed: %s\n", + platform_status_to_string(rc)); + goto cleanup; + } + + /* + * Exactly one reference per extent is mini_recover_allocations()'s own + * documented contract; reproducing a mini_allocator's full live-lifetime + * refcount (which may include self-references beyond this) is explicitly + * the job of a higher-level recovery walker, not this function. + */ + if (allocator_get_refcount((allocator *)&recovery, meta_head) + != AL_ONE_REF) + { + platform_error_log( + "cache_test: mini_recover_allocations did not recover the metadata " + "extent reference\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + for (uint64 i = 0; i < TEST_MINI_RECOVER_NUM_DATA_EXTENTS; i++) { + if (allocator_get_refcount((allocator *)&recovery, data_extents[i]) + != AL_ONE_REF) + { + platform_error_log("cache_test: mini_recover_allocations did not " + "recover data extent %lu\n", + i); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + } + rc_allocator_rebuild_finish(&recovery); + +cleanup: + if (mini_live) { + mini_release(&mini); + } + if (recovery_cc_live) { + clockcache_deinit(&recovery_cc); + } + if (recovery_al_live) { + if (recovery.recovery_in_progress) { + rc_allocator_abort_recovery(&recovery); + } else { + rc_allocator_deinit(&recovery); + } + } + if (original_cc_live) { + clockcache_deinit(&original_cc); + } + if (original_al_live) { + rc_allocator_deinit(&original); + } + + if (SUCCESS(rc)) { + platform_default_log( + "cache_test: mini_recover_allocations test passed\n"); + } + return rc; +} + typedef struct { cache *cc; volatile bool32 finished; @@ -1812,6 +1988,12 @@ cache_test(int argc, char *argv[]) goto destroy_iohandle; } + rc = test_mini_recover_allocations( + &system_cfg.allocator_cfg, &system_cfg.cache_cfg, io, hid); + if (!SUCCESS(rc)) { + goto destroy_iohandle; + } + rc = test_init_task_system(&ts, hid, &system_cfg.task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n",