Skip to content

Commit 3415426

Browse files
libsql-server: more namespace store tests around emptied entries
- lookups racing a destroy do not resurrect the namespace (no record, directory or filled entry afterwards; 20 rounds of 8 lookups); - reset and destroy of a namespace whose entry was left empty; - a create and a fork racing for one destination: the fork waits for the create's reservation, then is rejected, and leaves nothing behind; - lazy creation (what a replica runs with): an evicted namespace is reloaded and giving up is "unavailable", never "doesn't exist"; - a 10 s randomized run (`#[ignore]`): 12 tasks create, fork, look up, checkpoint and evict up to 60 namespaces; no lookup of a created namespace may fail, and afterwards every namespace is servable, no unlocked empty entry is cached, nothing is pinned, no marker remains. The randomized run leaves `destroy` out: a lookup whose entry was evicted can be loading while `destroy` removes the directory, which fails the destroy with `Directory not empty` and can leave a loaded namespace cached for a name the metastore no longer knows. That predates this change. It also keeps evictions rare: a lookup whose entry is emptied again after each of its three loads gives up with `NamespaceUnavailable` by design. `make_store` takes the lazy-creation flag and the cache capacity.
1 parent f1dc57c commit 3415426

1 file changed

Lines changed: 356 additions & 2 deletions

File tree

‎libsql-server/src/namespace/store.rs‎

Lines changed: 356 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1043,6 +1043,16 @@ pub(crate) mod test {
10431043
use super::*;
10441044

10451045
pub(crate) async fn make_store(path: &Path) -> NamespaceStore {
1046+
make_store_with(path, false, 10).await
1047+
}
1048+
1049+
/// `allow_lazy_creation` is what a replica runs with: a lookup of an unknown namespace
1050+
/// creates it. `max_active_namespaces` is the cache's capacity.
1051+
async fn make_store_with(
1052+
path: &Path,
1053+
allow_lazy_creation: bool,
1054+
max_active_namespaces: usize,
1055+
) -> NamespaceStore {
10461056
let (maker, manager) = metastore_connection_maker(None, path).await.unwrap();
10471057
let meta_store = MetaStore::new(
10481058
Default::default(),
@@ -1080,9 +1090,9 @@ pub(crate) mod test {
10801090
Arc::new(libsql_sys::wal::Sqlite3WalManager::default),
10811091
));
10821092
NamespaceStore::new(
1093+
allow_lazy_creation,
10831094
false,
1084-
false,
1085-
10,
1095+
max_active_namespaces,
10861096
meta_store,
10871097
configurators,
10881098
DatabaseKind::Primary,
@@ -1946,6 +1956,350 @@ pub(crate) mod test {
19461956
assert!(!store.exists(&foo).await);
19471957
}
19481958

1959+
/// Lookups racing a destroy must not bring the namespace back: no record, no directory,
1960+
/// no filled cache entry once both are done. The window is between a lookup's metastore
1961+
/// check and its load; it cannot be hit deterministically, so this hammers it.
1962+
#[tokio::test]
1963+
async fn lookups_racing_a_destroy_do_not_resurrect_the_namespace() {
1964+
let tmp = tempdir().unwrap();
1965+
let store = make_store(tmp.path()).await;
1966+
let foo = NamespaceName::from_string("foo".into()).unwrap();
1967+
1968+
for _ in 0..20 {
1969+
create(&store, "foo").await;
1970+
let destroyed = Arc::new(AtomicBool::new(false));
1971+
let lookups: Vec<_> = (0..8)
1972+
.map(|_| {
1973+
tokio::spawn({
1974+
let (store, foo, destroyed) =
1975+
(store.clone(), foo.clone(), destroyed.clone());
1976+
async move {
1977+
let mut after_destroy_ok = 0;
1978+
while !destroyed.load(Ordering::Relaxed) {
1979+
let was_destroyed = destroyed.load(Ordering::Relaxed);
1980+
let result = store.with(foo.clone(), |_| ()).await;
1981+
match result {
1982+
Ok(()) if was_destroyed => after_destroy_ok += 1,
1983+
Ok(()) => {}
1984+
Err(e) => assert!(
1985+
matches!(e.as_ref(), Error::NamespaceDoesntExist(_)),
1986+
"{e:?}"
1987+
),
1988+
}
1989+
tokio::task::yield_now().await;
1990+
}
1991+
after_destroy_ok
1992+
}
1993+
})
1994+
})
1995+
.collect();
1996+
tokio::task::yield_now().await;
1997+
store.destroy(foo.clone(), true).await.unwrap();
1998+
destroyed.store(true, Ordering::Relaxed);
1999+
for lookup in lookups {
2000+
assert_eq!(lookup.await.unwrap(), 0, "a lookup served after destroy");
2001+
}
2002+
2003+
assert!(!store.exists(&foo).await);
2004+
assert!(!tmp.path().join("dbs").join("foo").exists());
2005+
if let Some(entry) = store.inner.store.get(&foo).await {
2006+
assert!(entry.read().await.is_none(), "a namespace was resurrected");
2007+
}
2008+
store.inner.store.invalidate(&foo).await;
2009+
assert!(store.inner.in_flight.lock().is_empty());
2010+
}
2011+
}
2012+
2013+
#[tokio::test]
2014+
async fn reset_of_a_namespace_whose_entry_was_left_empty() {
2015+
let tmp = tempdir().unwrap();
2016+
let store = make_store(tmp.path()).await;
2017+
let foo = create(&store, "foo").await;
2018+
let entry = store.inner.store.get(&foo).await.unwrap();
2019+
entry
2020+
.write()
2021+
.await
2022+
.take()
2023+
.unwrap()
2024+
.shutdown(false)
2025+
.await
2026+
.unwrap();
2027+
2028+
store
2029+
.reset(foo.clone(), RestoreOption::Latest)
2030+
.await
2031+
.unwrap();
2032+
2033+
let cached = store.inner.store.get(&foo).await.unwrap();
2034+
assert!(cached.read().await.is_some(), "reset installed nothing");
2035+
assert!(store.inner.in_flight.lock().is_empty());
2036+
store.with(foo, |_| ()).await.unwrap();
2037+
}
2038+
2039+
#[tokio::test]
2040+
async fn create_and_fork_racing_for_one_destination() {
2041+
let tmp = tempdir().unwrap();
2042+
let store = make_store(tmp.path()).await;
2043+
let src = create(&store, "src").await;
2044+
let dst = NamespaceName::from_string("dst".into()).unwrap();
2045+
2046+
let (tx, dump) = slow_dump();
2047+
let create = tokio::spawn({
2048+
let (store, dst) = (store.clone(), dst.clone());
2049+
async move {
2050+
store
2051+
.create(dst, RestoreOption::Dump(dump), Default::default())
2052+
.await
2053+
}
2054+
});
2055+
tx.send(Ok(Bytes::from_static(
2056+
b"BEGIN TRANSACTION; CREATE TABLE test (x); INSERT INTO test VALUES (1);",
2057+
)))
2058+
.await
2059+
.unwrap();
2060+
wait_until_in_flight(&store, &dst).await;
2061+
let fork = tokio::spawn({
2062+
let (store, src, dst) = (store.clone(), src.clone(), dst.clone());
2063+
async move { store.fork(src, dst, Default::default(), None).await }
2064+
});
2065+
// The fork waits for the reservation the create holds.
2066+
tokio::time::sleep(Duration::from_millis(50)).await;
2067+
assert!(!fork.is_finished());
2068+
tx.send(Ok(Bytes::from_static(b"COMMIT;"))).await.unwrap();
2069+
drop(tx);
2070+
2071+
create.await.unwrap().unwrap();
2072+
let error = fork.await.unwrap().unwrap_err();
2073+
assert!(
2074+
matches!(error.as_ref(), Error::NamespaceAlreadyExist(_)),
2075+
"{error:?}"
2076+
);
2077+
// The create's result is what is served, and the rejected fork left nothing behind.
2078+
assert_eq!(count_test_rows(&store, "dst").await.unwrap(), 1);
2079+
assert!(store.inner.in_flight.lock().is_empty());
2080+
assert!(!tmp
2081+
.path()
2082+
.join("dbs")
2083+
.join("dst")
2084+
.join(".incomplete")
2085+
.exists());
2086+
}
2087+
2088+
#[tokio::test]
2089+
async fn destroy_of_a_namespace_whose_entry_was_left_empty() {
2090+
let tmp = tempdir().unwrap();
2091+
let store = make_store(tmp.path()).await;
2092+
let foo = create(&store, "foo").await;
2093+
let entry = store.inner.store.get(&foo).await.unwrap();
2094+
entry
2095+
.write()
2096+
.await
2097+
.take()
2098+
.unwrap()
2099+
.shutdown(false)
2100+
.await
2101+
.unwrap();
2102+
2103+
store.destroy(foo.clone(), true).await.unwrap();
2104+
2105+
assert!(!store.exists(&foo).await);
2106+
assert!(!tmp.path().join("dbs").join("foo").exists());
2107+
assert!(store.inner.store.get(&foo).await.is_none());
2108+
let error = store.with(foo, |_| ()).await.unwrap_err();
2109+
assert!(
2110+
matches!(error.as_ref(), Error::NamespaceDoesntExist(_)),
2111+
"{error:?}"
2112+
);
2113+
}
2114+
2115+
/// On a replica every lookup may create the namespace, so no lookup ever answers "doesn't
2116+
/// exist", and an entry emptied while a request waits is reloaded like on a primary.
2117+
#[tokio::test]
2118+
async fn lazy_creation_reloads_an_evicted_namespace_and_never_reports_it_missing() {
2119+
let tmp = tempdir().unwrap();
2120+
let store = make_store_with(tmp.path(), true, 10).await;
2121+
let namespace = NamespaceName::from_string("ns".into()).unwrap();
2122+
// Created by the lookup itself.
2123+
store.with(namespace.clone(), |_| ()).await.unwrap();
2124+
assert!(store.exists(&namespace).await);
2125+
2126+
let entry = store.inner.store.get(&namespace).await.unwrap();
2127+
let mut lock = entry.write().await;
2128+
let request = start_waiting_request(&store, &namespace, &entry).await;
2129+
let eviction = tokio::spawn({
2130+
let store = store.clone();
2131+
let namespace = namespace.clone();
2132+
async move { store.inner.store.invalidate(&namespace).await }
2133+
});
2134+
wait_until("the entry is evicted", || {
2135+
!store.inner.store.contains_key(&namespace)
2136+
})
2137+
.await;
2138+
lock.take().unwrap().shutdown(false).await.unwrap();
2139+
drop(lock);
2140+
2141+
assert_eq!(request.await.unwrap().unwrap(), namespace);
2142+
eviction.await.unwrap();
2143+
assert!(store
2144+
.inner
2145+
.store
2146+
.get(&namespace)
2147+
.await
2148+
.unwrap()
2149+
.read()
2150+
.await
2151+
.is_some());
2152+
2153+
// With the entry left empty for good, giving up is "unavailable", not "missing".
2154+
let entry = store.inner.store.get(&namespace).await.unwrap();
2155+
entry
2156+
.write()
2157+
.await
2158+
.take()
2159+
.unwrap()
2160+
.shutdown(false)
2161+
.await
2162+
.unwrap();
2163+
let error = store.emptied_entry_error(&namespace).await;
2164+
assert!(matches!(error, Error::NamespaceUnavailable(_)), "{error:?}");
2165+
}
2166+
2167+
/// Random `with`, `create`, `fork`, `checkpoint` and eviction for a while, on up to 60
2168+
/// namespaces, with one operation in fifty evicting an entry, so that entries are emptied
2169+
/// while in use all the time. Nothing is destroyed, so every "doesn't exist" is a false one.
2170+
///
2171+
/// Checked throughout: a lookup of a created namespace never fails, and a create or fork is
2172+
/// either accepted or rejected with "already exists". Checked at the end: every created
2173+
/// namespace is servable, no unlocked empty entry is left, nothing is pinned, and no
2174+
/// `.incomplete` marker remains.
2175+
///
2176+
/// `destroy` is left out on purpose: a lookup whose entry was evicted may be loading
2177+
/// (creating files) while `destroy` removes the directory, which fails the destroy with
2178+
/// `Directory not empty` and can leave a loaded namespace cached for a name the metastore
2179+
/// no longer knows. That predates this change and is not fixed by it.
2180+
///
2181+
/// The cache is larger than the number of namespaces, and evictions are rare, on purpose.
2182+
/// A lookup gives up with `NamespaceUnavailable` when its entry is emptied again after each
2183+
/// of `MAX_EMPTIED_ENTRY_ATTEMPTS` loads. A full cache does that (moka's admission policy
2184+
/// can evict a freshly loaded entry at once), and so does evicting one name from twelve
2185+
/// tasks faster than a load takes. Both are the designed outcome of eviction outrunning
2186+
/// loads (the base answered "doesn't exist" after the first load), not what this test is
2187+
/// about.
2188+
///
2189+
/// Slow, so not part of the default run:
2190+
/// `cargo nextest run -p libsql-server --run-ignored only -E 'test(random_operations)'`.
2191+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2192+
#[ignore]
2193+
async fn random_operations_keep_the_store_consistent() {
2194+
use std::sync::atomic::AtomicUsize;
2195+
2196+
let tmp = tempdir().unwrap();
2197+
const NAMES: usize = 60;
2198+
let store = make_store_with(tmp.path(), false, 1000).await;
2199+
let names: Vec<NamespaceName> = (0..NAMES)
2200+
.map(|i| NamespaceName::from_string(format!("ns{i}")).unwrap())
2201+
.collect();
2202+
// Number of namespaces known to exist; `names[..created]` are safe to look up.
2203+
for name in &names[..10] {
2204+
create(&store, name.as_str()).await;
2205+
}
2206+
let created = Arc::new(AtomicUsize::new(10));
2207+
let deadline = Instant::now() + Duration::from_secs(10);
2208+
2209+
let workers: Vec<_> = (0..12)
2210+
.map(|worker| {
2211+
tokio::spawn({
2212+
let (store, names, created) = (store.clone(), names.clone(), created.clone());
2213+
async move {
2214+
let mut seed = 0x9E3779B97F4A7C15u64 ^ (worker as u64 + 1);
2215+
let mut next = || {
2216+
seed ^= seed << 13;
2217+
seed ^= seed >> 7;
2218+
seed ^= seed << 17;
2219+
seed
2220+
};
2221+
let mut ops = 0u64;
2222+
while Instant::now() < deadline {
2223+
let known = created.load(Ordering::SeqCst);
2224+
let pick = |next: &mut dyn FnMut() -> u64| {
2225+
names[(next() % known.max(1) as u64) as usize].clone()
2226+
};
2227+
match next() % 50 {
2228+
0..=9 if known < NAMES => {
2229+
// Create or fork the next name; whoever is second gets
2230+
// "already exists". Only a success makes it known.
2231+
let name = names[known].clone();
2232+
let r = if known > 0 && next() % 2 == 0 {
2233+
store
2234+
.fork(pick(&mut next), name, Default::default(), None)
2235+
.await
2236+
} else {
2237+
store
2238+
.create(name, RestoreOption::Latest, Default::default())
2239+
.await
2240+
};
2241+
match r {
2242+
Ok(()) => {
2243+
created.store(known + 1, Ordering::SeqCst);
2244+
}
2245+
Err(e) => assert!(
2246+
matches!(e.as_ref(), Error::NamespaceAlreadyExist(_)),
2247+
"create/fork: {e:?}"
2248+
),
2249+
}
2250+
}
2251+
10 if known > 0 => {
2252+
store.inner.store.invalidate(&pick(&mut next)).await
2253+
}
2254+
11..=15 if known > 0 => {
2255+
// May fail with SQLITE_BUSY under concurrent writes.
2256+
let _ = store.checkpoint(pick(&mut next)).await;
2257+
}
2258+
_ if known > 0 => {
2259+
let name = pick(&mut next);
2260+
if let Err(e) = store.with(name.clone(), |_| ()).await {
2261+
panic!("lookup of {name}: {e:?}");
2262+
}
2263+
}
2264+
_ => tokio::task::yield_now().await,
2265+
}
2266+
ops += 1;
2267+
}
2268+
ops
2269+
}
2270+
})
2271+
})
2272+
.collect();
2273+
let mut ops = 0;
2274+
for worker in workers {
2275+
ops += worker.await.unwrap();
2276+
}
2277+
let known = created.load(Ordering::SeqCst);
2278+
eprintln!("{ops} operations on {known} namespaces");
2279+
assert!(
2280+
ops > 1000 && known > 10,
2281+
"only {ops} operations, {known} namespaces"
2282+
);
2283+
2284+
store.inner.store.run_pending_tasks().await;
2285+
assert!(store.inner.in_flight.lock().is_empty());
2286+
for name in &names[..known] {
2287+
store.with(name.clone(), |_| ()).await.unwrap();
2288+
assert!(!tmp
2289+
.path()
2290+
.join("dbs")
2291+
.join(name.as_str())
2292+
.join(".incomplete")
2293+
.exists());
2294+
}
2295+
for (name, entry) in store.inner.store.iter() {
2296+
let lock = entry
2297+
.try_read()
2298+
.expect("nothing holds an entry locked any more");
2299+
assert!(lock.is_some(), "{name}: an unlocked empty entry was left");
2300+
}
2301+
}
2302+
19492303
#[tokio::test]
19502304
async fn create_after_shutdown_is_refused() {
19512305
let tmp = tempdir().unwrap();

0 commit comments

Comments
 (0)