@@ -1043,6 +1043,16 @@ pub(crate) mod test {
10431043 use super :: * ;
10441044
10451045 pub ( crate ) async fn make_store ( path : & Path ) -> NamespaceStore {
1046+ make_store_with ( path, false , 10 ) . await
1047+ }
1048+
1049+ /// `allow_lazy_creation` is what a replica runs with: a lookup of an unknown namespace
1050+ /// creates it. `max_active_namespaces` is the cache's capacity.
1051+ async fn make_store_with (
1052+ path : & Path ,
1053+ allow_lazy_creation : bool ,
1054+ max_active_namespaces : usize ,
1055+ ) -> NamespaceStore {
10461056 let ( maker, manager) = metastore_connection_maker ( None , path) . await . unwrap ( ) ;
10471057 let meta_store = MetaStore :: new (
10481058 Default :: default ( ) ,
@@ -1080,9 +1090,9 @@ pub(crate) mod test {
10801090 Arc :: new ( libsql_sys:: wal:: Sqlite3WalManager :: default) ,
10811091 ) ) ;
10821092 NamespaceStore :: new (
1093+ allow_lazy_creation,
10831094 false ,
1084- false ,
1085- 10 ,
1095+ max_active_namespaces,
10861096 meta_store,
10871097 configurators,
10881098 DatabaseKind :: Primary ,
@@ -1946,6 +1956,350 @@ pub(crate) mod test {
19461956 assert ! ( !store. exists( & foo) . await ) ;
19471957 }
19481958
1959+ /// Lookups racing a destroy must not bring the namespace back: no record, no directory,
1960+ /// no filled cache entry once both are done. The window is between a lookup's metastore
1961+ /// check and its load; it cannot be hit deterministically, so this hammers it.
1962+ #[ tokio:: test]
1963+ async fn lookups_racing_a_destroy_do_not_resurrect_the_namespace ( ) {
1964+ let tmp = tempdir ( ) . unwrap ( ) ;
1965+ let store = make_store ( tmp. path ( ) ) . await ;
1966+ let foo = NamespaceName :: from_string ( "foo" . into ( ) ) . unwrap ( ) ;
1967+
1968+ for _ in 0 ..20 {
1969+ create ( & store, "foo" ) . await ;
1970+ let destroyed = Arc :: new ( AtomicBool :: new ( false ) ) ;
1971+ let lookups: Vec < _ > = ( 0 ..8 )
1972+ . map ( |_| {
1973+ tokio:: spawn ( {
1974+ let ( store, foo, destroyed) =
1975+ ( store. clone ( ) , foo. clone ( ) , destroyed. clone ( ) ) ;
1976+ async move {
1977+ let mut after_destroy_ok = 0 ;
1978+ while !destroyed. load ( Ordering :: Relaxed ) {
1979+ let was_destroyed = destroyed. load ( Ordering :: Relaxed ) ;
1980+ let result = store. with ( foo. clone ( ) , |_| ( ) ) . await ;
1981+ match result {
1982+ Ok ( ( ) ) if was_destroyed => after_destroy_ok += 1 ,
1983+ Ok ( ( ) ) => { }
1984+ Err ( e) => assert ! (
1985+ matches!( e. as_ref( ) , Error :: NamespaceDoesntExist ( _) ) ,
1986+ "{e:?}"
1987+ ) ,
1988+ }
1989+ tokio:: task:: yield_now ( ) . await ;
1990+ }
1991+ after_destroy_ok
1992+ }
1993+ } )
1994+ } )
1995+ . collect ( ) ;
1996+ tokio:: task:: yield_now ( ) . await ;
1997+ store. destroy ( foo. clone ( ) , true ) . await . unwrap ( ) ;
1998+ destroyed. store ( true , Ordering :: Relaxed ) ;
1999+ for lookup in lookups {
2000+ assert_eq ! ( lookup. await . unwrap( ) , 0 , "a lookup served after destroy" ) ;
2001+ }
2002+
2003+ assert ! ( !store. exists( & foo) . await ) ;
2004+ assert ! ( !tmp. path( ) . join( "dbs" ) . join( "foo" ) . exists( ) ) ;
2005+ if let Some ( entry) = store. inner . store . get ( & foo) . await {
2006+ assert ! ( entry. read( ) . await . is_none( ) , "a namespace was resurrected" ) ;
2007+ }
2008+ store. inner . store . invalidate ( & foo) . await ;
2009+ assert ! ( store. inner. in_flight. lock( ) . is_empty( ) ) ;
2010+ }
2011+ }
2012+
2013+ #[ tokio:: test]
2014+ async fn reset_of_a_namespace_whose_entry_was_left_empty ( ) {
2015+ let tmp = tempdir ( ) . unwrap ( ) ;
2016+ let store = make_store ( tmp. path ( ) ) . await ;
2017+ let foo = create ( & store, "foo" ) . await ;
2018+ let entry = store. inner . store . get ( & foo) . await . unwrap ( ) ;
2019+ entry
2020+ . write ( )
2021+ . await
2022+ . take ( )
2023+ . unwrap ( )
2024+ . shutdown ( false )
2025+ . await
2026+ . unwrap ( ) ;
2027+
2028+ store
2029+ . reset ( foo. clone ( ) , RestoreOption :: Latest )
2030+ . await
2031+ . unwrap ( ) ;
2032+
2033+ let cached = store. inner . store . get ( & foo) . await . unwrap ( ) ;
2034+ assert ! ( cached. read( ) . await . is_some( ) , "reset installed nothing" ) ;
2035+ assert ! ( store. inner. in_flight. lock( ) . is_empty( ) ) ;
2036+ store. with ( foo, |_| ( ) ) . await . unwrap ( ) ;
2037+ }
2038+
2039+ #[ tokio:: test]
2040+ async fn create_and_fork_racing_for_one_destination ( ) {
2041+ let tmp = tempdir ( ) . unwrap ( ) ;
2042+ let store = make_store ( tmp. path ( ) ) . await ;
2043+ let src = create ( & store, "src" ) . await ;
2044+ let dst = NamespaceName :: from_string ( "dst" . into ( ) ) . unwrap ( ) ;
2045+
2046+ let ( tx, dump) = slow_dump ( ) ;
2047+ let create = tokio:: spawn ( {
2048+ let ( store, dst) = ( store. clone ( ) , dst. clone ( ) ) ;
2049+ async move {
2050+ store
2051+ . create ( dst, RestoreOption :: Dump ( dump) , Default :: default ( ) )
2052+ . await
2053+ }
2054+ } ) ;
2055+ tx. send ( Ok ( Bytes :: from_static (
2056+ b"BEGIN TRANSACTION; CREATE TABLE test (x); INSERT INTO test VALUES (1);" ,
2057+ ) ) )
2058+ . await
2059+ . unwrap ( ) ;
2060+ wait_until_in_flight ( & store, & dst) . await ;
2061+ let fork = tokio:: spawn ( {
2062+ let ( store, src, dst) = ( store. clone ( ) , src. clone ( ) , dst. clone ( ) ) ;
2063+ async move { store. fork ( src, dst, Default :: default ( ) , None ) . await }
2064+ } ) ;
2065+ // The fork waits for the reservation the create holds.
2066+ tokio:: time:: sleep ( Duration :: from_millis ( 50 ) ) . await ;
2067+ assert ! ( !fork. is_finished( ) ) ;
2068+ tx. send ( Ok ( Bytes :: from_static ( b"COMMIT;" ) ) ) . await . unwrap ( ) ;
2069+ drop ( tx) ;
2070+
2071+ create. await . unwrap ( ) . unwrap ( ) ;
2072+ let error = fork. await . unwrap ( ) . unwrap_err ( ) ;
2073+ assert ! (
2074+ matches!( error. as_ref( ) , Error :: NamespaceAlreadyExist ( _) ) ,
2075+ "{error:?}"
2076+ ) ;
2077+ // The create's result is what is served, and the rejected fork left nothing behind.
2078+ assert_eq ! ( count_test_rows( & store, "dst" ) . await . unwrap( ) , 1 ) ;
2079+ assert ! ( store. inner. in_flight. lock( ) . is_empty( ) ) ;
2080+ assert ! ( !tmp
2081+ . path( )
2082+ . join( "dbs" )
2083+ . join( "dst" )
2084+ . join( ".incomplete" )
2085+ . exists( ) ) ;
2086+ }
2087+
2088+ #[ tokio:: test]
2089+ async fn destroy_of_a_namespace_whose_entry_was_left_empty ( ) {
2090+ let tmp = tempdir ( ) . unwrap ( ) ;
2091+ let store = make_store ( tmp. path ( ) ) . await ;
2092+ let foo = create ( & store, "foo" ) . await ;
2093+ let entry = store. inner . store . get ( & foo) . await . unwrap ( ) ;
2094+ entry
2095+ . write ( )
2096+ . await
2097+ . take ( )
2098+ . unwrap ( )
2099+ . shutdown ( false )
2100+ . await
2101+ . unwrap ( ) ;
2102+
2103+ store. destroy ( foo. clone ( ) , true ) . await . unwrap ( ) ;
2104+
2105+ assert ! ( !store. exists( & foo) . await ) ;
2106+ assert ! ( !tmp. path( ) . join( "dbs" ) . join( "foo" ) . exists( ) ) ;
2107+ assert ! ( store. inner. store. get( & foo) . await . is_none( ) ) ;
2108+ let error = store. with ( foo, |_| ( ) ) . await . unwrap_err ( ) ;
2109+ assert ! (
2110+ matches!( error. as_ref( ) , Error :: NamespaceDoesntExist ( _) ) ,
2111+ "{error:?}"
2112+ ) ;
2113+ }
2114+
2115+ /// On a replica every lookup may create the namespace, so no lookup ever answers "doesn't
2116+ /// exist", and an entry emptied while a request waits is reloaded like on a primary.
2117+ #[ tokio:: test]
2118+ async fn lazy_creation_reloads_an_evicted_namespace_and_never_reports_it_missing ( ) {
2119+ let tmp = tempdir ( ) . unwrap ( ) ;
2120+ let store = make_store_with ( tmp. path ( ) , true , 10 ) . await ;
2121+ let namespace = NamespaceName :: from_string ( "ns" . into ( ) ) . unwrap ( ) ;
2122+ // Created by the lookup itself.
2123+ store. with ( namespace. clone ( ) , |_| ( ) ) . await . unwrap ( ) ;
2124+ assert ! ( store. exists( & namespace) . await ) ;
2125+
2126+ let entry = store. inner . store . get ( & namespace) . await . unwrap ( ) ;
2127+ let mut lock = entry. write ( ) . await ;
2128+ let request = start_waiting_request ( & store, & namespace, & entry) . await ;
2129+ let eviction = tokio:: spawn ( {
2130+ let store = store. clone ( ) ;
2131+ let namespace = namespace. clone ( ) ;
2132+ async move { store. inner . store . invalidate ( & namespace) . await }
2133+ } ) ;
2134+ wait_until ( "the entry is evicted" , || {
2135+ !store. inner . store . contains_key ( & namespace)
2136+ } )
2137+ . await ;
2138+ lock. take ( ) . unwrap ( ) . shutdown ( false ) . await . unwrap ( ) ;
2139+ drop ( lock) ;
2140+
2141+ assert_eq ! ( request. await . unwrap( ) . unwrap( ) , namespace) ;
2142+ eviction. await . unwrap ( ) ;
2143+ assert ! ( store
2144+ . inner
2145+ . store
2146+ . get( & namespace)
2147+ . await
2148+ . unwrap( )
2149+ . read( )
2150+ . await
2151+ . is_some( ) ) ;
2152+
2153+ // With the entry left empty for good, giving up is "unavailable", not "missing".
2154+ let entry = store. inner . store . get ( & namespace) . await . unwrap ( ) ;
2155+ entry
2156+ . write ( )
2157+ . await
2158+ . take ( )
2159+ . unwrap ( )
2160+ . shutdown ( false )
2161+ . await
2162+ . unwrap ( ) ;
2163+ let error = store. emptied_entry_error ( & namespace) . await ;
2164+ assert ! ( matches!( error, Error :: NamespaceUnavailable ( _) ) , "{error:?}" ) ;
2165+ }
2166+
2167+ /// Random `with`, `create`, `fork`, `checkpoint` and eviction for a while, on up to 60
2168+ /// namespaces, with one operation in fifty evicting an entry, so that entries are emptied
2169+ /// while in use all the time. Nothing is destroyed, so every "doesn't exist" is a false one.
2170+ ///
2171+ /// Checked throughout: a lookup of a created namespace never fails, and a create or fork is
2172+ /// either accepted or rejected with "already exists". Checked at the end: every created
2173+ /// namespace is servable, no unlocked empty entry is left, nothing is pinned, and no
2174+ /// `.incomplete` marker remains.
2175+ ///
2176+ /// `destroy` is left out on purpose: a lookup whose entry was evicted may be loading
2177+ /// (creating files) while `destroy` removes the directory, which fails the destroy with
2178+ /// `Directory not empty` and can leave a loaded namespace cached for a name the metastore
2179+ /// no longer knows. That predates this change and is not fixed by it.
2180+ ///
2181+ /// The cache is larger than the number of namespaces, and evictions are rare, on purpose.
2182+ /// A lookup gives up with `NamespaceUnavailable` when its entry is emptied again after each
2183+ /// of `MAX_EMPTIED_ENTRY_ATTEMPTS` loads. A full cache does that (moka's admission policy
2184+ /// can evict a freshly loaded entry at once), and so does evicting one name from twelve
2185+ /// tasks faster than a load takes. Both are the designed outcome of eviction outrunning
2186+ /// loads (the base answered "doesn't exist" after the first load), not what this test is
2187+ /// about.
2188+ ///
2189+ /// Slow, so not part of the default run:
2190+ /// `cargo nextest run -p libsql-server --run-ignored only -E 'test(random_operations)'`.
2191+ #[ tokio:: test( flavor = "multi_thread" , worker_threads = 4 ) ]
2192+ #[ ignore]
2193+ async fn random_operations_keep_the_store_consistent ( ) {
2194+ use std:: sync:: atomic:: AtomicUsize ;
2195+
2196+ let tmp = tempdir ( ) . unwrap ( ) ;
2197+ const NAMES : usize = 60 ;
2198+ let store = make_store_with ( tmp. path ( ) , false , 1000 ) . await ;
2199+ let names: Vec < NamespaceName > = ( 0 ..NAMES )
2200+ . map ( |i| NamespaceName :: from_string ( format ! ( "ns{i}" ) ) . unwrap ( ) )
2201+ . collect ( ) ;
2202+ // Number of namespaces known to exist; `names[..created]` are safe to look up.
2203+ for name in & names[ ..10 ] {
2204+ create ( & store, name. as_str ( ) ) . await ;
2205+ }
2206+ let created = Arc :: new ( AtomicUsize :: new ( 10 ) ) ;
2207+ let deadline = Instant :: now ( ) + Duration :: from_secs ( 10 ) ;
2208+
2209+ let workers: Vec < _ > = ( 0 ..12 )
2210+ . map ( |worker| {
2211+ tokio:: spawn ( {
2212+ let ( store, names, created) = ( store. clone ( ) , names. clone ( ) , created. clone ( ) ) ;
2213+ async move {
2214+ let mut seed = 0x9E3779B97F4A7C15u64 ^ ( worker as u64 + 1 ) ;
2215+ let mut next = || {
2216+ seed ^= seed << 13 ;
2217+ seed ^= seed >> 7 ;
2218+ seed ^= seed << 17 ;
2219+ seed
2220+ } ;
2221+ let mut ops = 0u64 ;
2222+ while Instant :: now ( ) < deadline {
2223+ let known = created. load ( Ordering :: SeqCst ) ;
2224+ let pick = |next : & mut dyn FnMut ( ) -> u64 | {
2225+ names[ ( next ( ) % known. max ( 1 ) as u64 ) as usize ] . clone ( )
2226+ } ;
2227+ match next ( ) % 50 {
2228+ 0 ..=9 if known < NAMES => {
2229+ // Create or fork the next name; whoever is second gets
2230+ // "already exists". Only a success makes it known.
2231+ let name = names[ known] . clone ( ) ;
2232+ let r = if known > 0 && next ( ) % 2 == 0 {
2233+ store
2234+ . fork ( pick ( & mut next) , name, Default :: default ( ) , None )
2235+ . await
2236+ } else {
2237+ store
2238+ . create ( name, RestoreOption :: Latest , Default :: default ( ) )
2239+ . await
2240+ } ;
2241+ match r {
2242+ Ok ( ( ) ) => {
2243+ created. store ( known + 1 , Ordering :: SeqCst ) ;
2244+ }
2245+ Err ( e) => assert ! (
2246+ matches!( e. as_ref( ) , Error :: NamespaceAlreadyExist ( _) ) ,
2247+ "create/fork: {e:?}"
2248+ ) ,
2249+ }
2250+ }
2251+ 10 if known > 0 => {
2252+ store. inner . store . invalidate ( & pick ( & mut next) ) . await
2253+ }
2254+ 11 ..=15 if known > 0 => {
2255+ // May fail with SQLITE_BUSY under concurrent writes.
2256+ let _ = store. checkpoint ( pick ( & mut next) ) . await ;
2257+ }
2258+ _ if known > 0 => {
2259+ let name = pick ( & mut next) ;
2260+ if let Err ( e) = store. with ( name. clone ( ) , |_| ( ) ) . await {
2261+ panic ! ( "lookup of {name}: {e:?}" ) ;
2262+ }
2263+ }
2264+ _ => tokio:: task:: yield_now ( ) . await ,
2265+ }
2266+ ops += 1 ;
2267+ }
2268+ ops
2269+ }
2270+ } )
2271+ } )
2272+ . collect ( ) ;
2273+ let mut ops = 0 ;
2274+ for worker in workers {
2275+ ops += worker. await . unwrap ( ) ;
2276+ }
2277+ let known = created. load ( Ordering :: SeqCst ) ;
2278+ eprintln ! ( "{ops} operations on {known} namespaces" ) ;
2279+ assert ! (
2280+ ops > 1000 && known > 10 ,
2281+ "only {ops} operations, {known} namespaces"
2282+ ) ;
2283+
2284+ store. inner . store . run_pending_tasks ( ) . await ;
2285+ assert ! ( store. inner. in_flight. lock( ) . is_empty( ) ) ;
2286+ for name in & names[ ..known] {
2287+ store. with ( name. clone ( ) , |_| ( ) ) . await . unwrap ( ) ;
2288+ assert ! ( !tmp
2289+ . path( )
2290+ . join( "dbs" )
2291+ . join( name. as_str( ) )
2292+ . join( ".incomplete" )
2293+ . exists( ) ) ;
2294+ }
2295+ for ( name, entry) in store. inner . store . iter ( ) {
2296+ let lock = entry
2297+ . try_read ( )
2298+ . expect ( "nothing holds an entry locked any more" ) ;
2299+ assert ! ( lock. is_some( ) , "{name}: an unlocked empty entry was left" ) ;
2300+ }
2301+ }
2302+
19492303 #[ tokio:: test]
19502304 async fn create_after_shutdown_is_refused ( ) {
19512305 let tmp = tempdir ( ) . unwrap ( ) ;
0 commit comments