1111//! same controller.
1212
1313use std:: collections:: HashMap ;
14- use std:: sync:: atomic:: { AtomicBool , AtomicU64 , Ordering } ;
14+ use std:: sync:: atomic:: { AtomicBool , AtomicU64 , AtomicUsize , Ordering } ;
1515use std:: sync:: Arc ;
1616
1717use parking_lot:: Mutex ;
@@ -261,9 +261,10 @@ struct ReadLeaseSet {
261261/// ([`FenceController::acquire_read_lease`]).
262262pub struct ReadLease {
263263 controller : Arc < FenceController > ,
264- id : u64 ,
264+ /// `None` for a SQL lease taken on the fast path: counted, not registered.
265+ id : Option < u64 > ,
265266 kind : LeaseKind ,
266- cancelled : Arc < AtomicBool > ,
267+ cancelled : Option < Arc < AtomicBool > > ,
267268}
268269
269270impl ReadLease {
@@ -277,7 +278,9 @@ impl ReadLease {
277278
278279 /// Whether the read drain cancelled this lease's work at its deadline.
279280 pub fn cancelled_by_fence ( & self ) -> bool {
280- self . cancelled . load ( Ordering :: Acquire )
281+ self . cancelled
282+ . as_ref ( )
283+ . is_some_and ( |c| c. load ( Ordering :: Acquire ) )
281284 }
282285}
283286
@@ -293,7 +296,10 @@ impl std::fmt::Debug for ReadLease {
293296
294297impl Drop for ReadLease {
295298 fn drop ( & mut self ) {
296- self . controller . release_read_lease ( self . id ) ;
299+ match self . id {
300+ Some ( id) => self . controller . release_read_lease ( id) ,
301+ None => self . controller . release_fast_read ( ) ,
302+ }
297303 }
298304}
299305
@@ -309,6 +315,11 @@ pub struct FenceController {
309315 write_drains : Mutex < Vec < WriteDrainSource > > ,
310316 /// The read leases held on the namespace (section 9).
311317 read_leases : Mutex < ReadLeaseSet > ,
318+ /// Whether SQL reads may take the fast path: the gate is in a state from which a read fence
319+ /// is at least two transitions away (section 9.1). Kept in step with every publication.
320+ fast_reads : AtomicBool ,
321+ /// SQL read leases taken on the fast path and not released yet.
322+ fast_sql_reads : AtomicUsize ,
312323 /// Notified whenever a read lease is released.
313324 read_released : Notify ,
314325 #[ cfg( test) ]
@@ -328,17 +339,21 @@ impl FenceController {
328339 /// A controller whose gate starts from `fence`, as established by the metastore.
329340 pub fn new ( namespace : NamespaceName , fence : StoredFence ) -> Arc < Self > {
330341 let ( gate, _) = watch:: channel ( GateSnapshot :: new ( fence) ) ;
331- Arc :: new ( Self {
342+ let this = Arc :: new ( Self {
332343 namespace,
333344 transition_lock : Default :: default ( ) ,
334345 gate,
335346 write_queues : Mutex :: new ( Vec :: new ( ) ) ,
336347 write_drains : Mutex :: new ( Vec :: new ( ) ) ,
337348 read_leases : Mutex :: new ( ReadLeaseSet :: default ( ) ) ,
349+ fast_reads : AtomicBool :: new ( false ) ,
350+ fast_sql_reads : AtomicUsize :: new ( 0 ) ,
338351 read_released : Notify :: new ( ) ,
339352 #[ cfg( test) ]
340353 hooks : FenceTestHooks :: default ( ) ,
341- } )
354+ } ) ;
355+ this. refresh_fast_reads ( ) ;
356+ this
342357 }
343358
344359 /// A controller for a namespace without fence state.
@@ -418,12 +433,30 @@ impl FenceController {
418433 /// is seen here, and one that closes after it counts this lease and waits for it. `cancel`
419434 /// is how the read drain stops the work at its deadline: it must make the work end and
420435 /// drop the lease, never wait for it to end. The lease is released when dropped.
436+ ///
437+ /// Fast path (section 9.1): while no read fence can be closer than two transitions away, a
438+ /// normal SQL read is only counted, without the lock or `cancel`. A read fence still waits
439+ /// for it, but cannot cancel it at its deadline.
421440 pub fn acquire_read_lease (
422441 self : & Arc < Self > ,
423442 class : OperationClass ,
424443 kind : LeaseKind ,
425444 cancel : impl Fn ( ) + Send + Sync + ' static ,
426445 ) -> Result < ReadLease , FenceError > {
446+ if kind == LeaseKind :: Sql && class == OperationClass :: NormalRead {
447+ // Counted before the flag is read: a gate that turns the fast path off after this
448+ // check sees the count, and one that turned it off before is seen here.
449+ self . fast_sql_reads . fetch_add ( 1 , Ordering :: SeqCst ) ;
450+ if self . fast_reads . load ( Ordering :: SeqCst ) {
451+ return Ok ( ReadLease {
452+ controller : self . clone ( ) ,
453+ id : None ,
454+ kind,
455+ cancelled : None ,
456+ } ) ;
457+ }
458+ self . release_fast_read ( ) ;
459+ }
427460 let cancelled = Arc :: new ( AtomicBool :: new ( false ) ) ;
428461 let id = {
429462 let mut leases = self . read_leases . lock ( ) ;
@@ -442,16 +475,53 @@ impl FenceController {
442475 } ;
443476 Ok ( ReadLease {
444477 controller : self . clone ( ) ,
445- id,
478+ id : Some ( id ) ,
446479 kind,
447- cancelled,
480+ cancelled : Some ( cancelled ) ,
448481 } )
449482 }
450483
484+ fn release_fast_read ( & self ) {
485+ // Only a read drain waits for releases, and none can run while the fast path is on.
486+ if self . fast_sql_reads . fetch_sub ( 1 , Ordering :: SeqCst ) == 1
487+ && !self . fast_reads . load ( Ordering :: SeqCst )
488+ {
489+ self . read_released . notify_waiters ( ) ;
490+ }
491+ }
492+
493+ /// Turn the SQL read fast path on or off after a change of the gate. On only for
494+ /// `UNFENCED`, `RELEASED` and `TARGET_WRITABLE` with nothing installing, indeterminate or
495+ /// closing reads: every other state is at most one transition away from a read fence. And
496+ /// only while the gate admits normal reads at all, whatever else may deny them.
497+ fn refresh_fast_reads ( & self ) {
498+ let fast = {
499+ let gate = self . gate . borrow ( ) ;
500+ gate. installing . is_none ( )
501+ && gate. indeterminate . is_none ( )
502+ && gate. closing_reads . is_none ( )
503+ && gate. permits ( OperationClass :: NormalRead ) . is_ok ( )
504+ && matches ! (
505+ gate. state( ) ,
506+ FenceState :: Unfenced | FenceState :: Released | FenceState :: TargetWritable
507+ )
508+ } ;
509+ self . fast_reads . store ( fast, Ordering :: SeqCst ) ;
510+ }
511+
512+ /// The read leases registered under the lease lock (every lease but those on the fast path).
513+ #[ cfg( test) ]
514+ pub ( crate ) fn registered_read_leases ( & self ) -> usize {
515+ self . read_leases . lock ( ) . live . len ( )
516+ }
517+
451518 /// The read leases currently held, by kind.
452519 pub fn read_lease_counts ( & self ) -> ReadLeaseCounts {
453520 let leases = self . read_leases . lock ( ) ;
454- let mut counts = ReadLeaseCounts :: default ( ) ;
521+ let mut counts = ReadLeaseCounts {
522+ sql : self . fast_sql_reads . load ( Ordering :: SeqCst ) ,
523+ ..Default :: default ( )
524+ } ;
455525 for entry in leases. live . values ( ) {
456526 match entry. kind {
457527 LeaseKind :: Sql => counts. sql += 1 ,
@@ -575,6 +645,7 @@ impl FenceController {
575645 "published namespace fence gate"
576646 ) ;
577647 }
648+ self . refresh_fast_reads ( ) ;
578649 // After the gate is published, so that every woken writer re-checks against it.
579650 if generation_changed {
580651 self . write_queues . lock ( ) . retain ( WriteQueueWaker :: wake) ;
@@ -628,6 +699,7 @@ impl Transition {
628699 self . controller
629700 . gate
630701 . send_modify ( |gate| gate. closing_reads = Some ( key) ) ;
702+ self . controller . refresh_fast_reads ( ) ;
631703 tracing:: debug!(
632704 namespace = %self . controller. namespace,
633705 operation_id = %key. 0 ,
@@ -643,6 +715,7 @@ impl Transition {
643715 gate. closing_reads = None ;
644716 was_closing
645717 } ) ;
718+ self . controller . refresh_fast_reads ( ) ;
646719 }
647720
648721 /// Commit `request` in the metastore and publish the result.
@@ -1192,6 +1265,44 @@ pub(crate) mod tests {
11921265
11931266 /// Registered write queues are woken after every change of the write generation, and only
11941267 /// then; a waker whose manager is gone is forgotten.
1268+ /// The cost of admitting one read, on the SQL fast path and on the registered path that every
1269+ /// read took before it (and dumps and streams still take). Not a pass/fail test:
1270+ /// `cargo test -p libsql-server --release --lib -- --ignored --nocapture read_lease_cost`.
1271+ #[ test]
1272+ #[ ignore]
1273+ fn read_lease_cost ( ) {
1274+ const N : u32 = 1_000_000 ;
1275+ const THREADS : u32 = 8 ;
1276+ let controller = FenceController :: unfenced ( "ns" . into ( ) ) ;
1277+ for ( label, class, kind) in [
1278+ (
1279+ "fast path (SQL, unfenced)" ,
1280+ OperationClass :: NormalRead ,
1281+ LeaseKind :: Sql ,
1282+ ) ,
1283+ ( "registered" , OperationClass :: Stream , LeaseKind :: Dump ) ,
1284+ ] {
1285+ let started = std:: time:: Instant :: now ( ) ;
1286+ for _ in 0 ..N {
1287+ drop ( controller. acquire_read_lease ( class, kind, || ( ) ) . unwrap ( ) ) ;
1288+ }
1289+ let one = started. elapsed ( ) / N ;
1290+ let started = std:: time:: Instant :: now ( ) ;
1291+ std:: thread:: scope ( |scope| {
1292+ for _ in 0 ..THREADS {
1293+ scope. spawn ( || {
1294+ for _ in 0 ..N / THREADS {
1295+ drop ( controller. acquire_read_lease ( class, kind, || ( ) ) . unwrap ( ) ) ;
1296+ }
1297+ } ) ;
1298+ }
1299+ } ) ;
1300+ let contended = started. elapsed ( ) / N ;
1301+ println ! ( "{label}: {one:?} per read on one thread, {contended:?} per read across {THREADS} threads" ) ;
1302+ }
1303+ assert_eq ! ( controller. read_lease_counts( ) . total( ) , 0 ) ;
1304+ }
1305+
11951306 #[ tokio:: test]
11961307 async fn dead_write_queues_and_drains_are_dropped_on_registration ( ) {
11971308 let controller = FenceController :: unfenced ( "ns" . into ( ) ) ;
0 commit comments