chunk backups for easier resumes

.github/workflows/pr-check.yml

@@ -30,6 +30,8 @@ jobs:
 
       - name: Set up Android SDK
         uses: android-actions/setup-android@v3
+        with:
+          packages: 'platform-tools'
 
       - name: Set up Gradle
         uses: gradle/actions/setup-gradle@v4

README.md

@@ -29,14 +29,17 @@ This repo is open source, but is depended on by shipping Light Phone products -
   for examples.
 - [**`BackupDataSource`**](library/src/jvmSharedMain/kotlin/com/thelightphone/backup/RemoteBackup.kt) -
   the interface a host app implements to say *what* to back up (`getPathsToBackUp`,
-  `getFilesToBackUpForPath`, `readFile`, `hashForFile`). This library ships no implementation, it
-  only knows how to move bytes once told which ones.
+  `getFilesToBackUpForPath`, `readFile`, `hashForFile`, `getEarliestPossibleBackupDate`). This
+  library ships no implementation, it only knows how to move bytes once told which ones.
 - [**`BackupRunner`**](library/src/jvmSharedMain/kotlin/com/thelightphone/backup/RemoteBackup.kt) -
-  orchestrates one backup pass: walks each path, uploads what's changed since that path's last
-  backup, writes its checksum manifest, and records a `_meta` entry for any path that made progress
-  (these are used to determine time of last successful backup). Platform-agnostic - on Android it's
-  invoked by [`BackupWorker`](library/src/androidMain/kotlin/com/thelightphone/backup/BackupWorker.kt),
-  an Android WorkManager `CoroutineWorker`.
+  orchestrates one backup pass: for each path, splits the time since its last backup into bounded,
+  independently-completable windows (one day by default - see `chunkWindows`), uploads what's changed
+  within each window, writes its checksum manifest, and records a `_meta` entry for any window that
+  made progress (these are used to determine time of last successful backup). Windowing means a long
+  catch-up period doesn't have to be redone in full after a partial failure - only the windows after
+  the last one that completed do. Platform-agnostic - on Android it's invoked by
+  [`BackupWorker`](library/src/androidMain/kotlin/com/thelightphone/backup/BackupWorker.kt), an
+  Android WorkManager `CoroutineWorker`.
 - [**`RemoteAccessTokenProvider`** /
   **`StoredOAuthTokenProvider`**](library/src/jvmSharedMain/kotlin/com/thelightphone/backup/RemoteAccessTokenProvider.kt) -
   provider-agnostic OAuth access token caching, refresh, and persistence.

build.gradle.kts

@@ -11,7 +11,7 @@ plugins {
 
 allprojects {
     group = "com.thelightphone.backup"
-    version = "0.0.4"
+    version = "0.0.5"
 
     plugins.withId("com.vanniktech.maven.publish") {
         extensions.configure<MavenPublishBaseExtension> {

gradle/libs.versions.toml

@@ -7,7 +7,7 @@ kotlinx-coroutines = "1.10.2"
 kotlinx-io = "0.8.2"
 junit = "4.13.2"
 vanniktechMavenPublish = "0.35.0"
-toolmanager = "0.0.13"
+toolmanager = "0.0.14"
 ktor = "3.4.2"
 androidxWork = "2.10.0"
 

library/src/androidMain/AndroidManifest.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools">
+
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
+
+    <application>
+        <service
+            android:name="androidx.work.impl.foreground.SystemForegroundService"
+            android:foregroundServiceType="dataSync"
+            tools:node="merge" />
+    </application>
+
+</manifest>

library/src/androidMain/kotlin/com/thelightphone/backup/BackupWorker.kt

@@ -1,10 +1,16 @@
 package com.thelightphone.backup
 
+import android.app.Notification
+import android.app.NotificationChannel
+import android.app.NotificationManager
 import android.content.Context
+import android.content.pm.ServiceInfo
+import android.os.Build
 import androidx.work.Constraints
 import androidx.work.CoroutineWorker
 import androidx.work.ExistingPeriodicWorkPolicy
 import androidx.work.ExistingWorkPolicy
+import androidx.work.ForegroundInfo
 import androidx.work.ListenableWorker
 import androidx.work.NetworkType
 import androidx.work.OneTimeWorkRequestBuilder
@@ -13,13 +19,16 @@ import androidx.work.PeriodicWorkRequestBuilder
 import androidx.work.WorkManager
 import androidx.work.WorkerFactory
 import androidx.work.WorkerParameters
+import androidx.work.workDataOf
 import kotlin.time.Clock
 import kotlin.time.Duration.Companion.milliseconds
 import kotlin.time.Duration.Companion.minutes
-import kotlin.time.Instant
 import kotlin.time.toJavaDuration
 
-// Runs one BackupRunner pass
+// Runs one BackupRunner pass. Always promoted to a foreground service (see buildForegroundInfo) -
+// an initial backup can run long enough that the OS would otherwise kill a plain background
+// worker partway through. LightOS's Android fork doesn't render the notification a foreground
+// service is normally required to show, so unlike a typical app this costs nothing in visible UI.
 class BackupWorker(
     context: Context,
     params: WorkerParameters,
@@ -32,8 +41,23 @@ class BackupWorker(
         val provider = preferences.getActiveProvider() ?: return Result.success()
         val remoteBackup = buildRemoteBackup(provider) ?: return Result.failure()
 
-        val runner = BackupRunner(remoteBackup, dependencyProvider.createDataSource(), clock)
-        return when (val result = runner.run(todayDirectoryName(clock.now()))) {
+        ensureNotificationChannel()
+        setForeground(buildForegroundInfo())
+
+        val runner = BackupRunner(
+            remoteBackup,
+            dependencyProvider.createDataSource(),
+            clock,
+            onProgress = { progress ->
+                setProgress(
+                    workDataOf(
+                        PROGRESS_WINDOWS_COMPLETED to progress.windowsCompleted,
+                        PROGRESS_TOTAL_WINDOWS to progress.totalWindows,
+                    ),
+                )
+            },
+        )
+        return when (val result = runner.run()) {
             is BackupResult.Completed -> {
                 preferences.setLastBackupStatus(BackupStatus.Succeeded(clock.now()))
                 Result.success()
@@ -44,6 +68,24 @@ class BackupWorker(
         }
     }
 
+    private fun ensureNotificationChannel() {
+        val channel = NotificationChannel(NOTIFICATION_CHANNEL_ID, "Backup", NotificationManager.IMPORTANCE_LOW)
+        applicationContext.getSystemService(NotificationManager::class.java).createNotificationChannel(channel)
+    }
+
+    private fun buildForegroundInfo(): ForegroundInfo {
+        val notification = Notification.Builder(applicationContext, NOTIFICATION_CHANNEL_ID)
+            .setContentTitle("LightOS Backup")
+            .setSmallIcon(android.R.drawable.stat_sys_upload)
+            .setOngoing(true)
+            .build()
+        return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+            ForegroundInfo(NOTIFICATION_ID, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
+        } else {
+            ForegroundInfo(NOTIFICATION_ID, notification)
+        }
+    }
+
     private suspend fun recordFailureAndDecide(cause: Throwable?): Result = when (cause) {
         is RemoteBackupError.Unauthorized -> {
             preferences.setLastBackupStatus(BackupStatus.NeedsReauth)
@@ -74,11 +116,17 @@ class BackupWorker(
             )
         }
     }
-}
 
-// "yyyy-MM-ddTHH-mm-ssZ"
-private fun todayDirectoryName(now: Instant): String =
-    Instant.fromEpochSeconds(now.epochSeconds).toString().replace(":", "-")
+    companion object {
+        // Keys into the Data reported via CoroutineWorker.setProgress(), observable through
+        // WorkInfo.progress - values are BackupProgress.windowsCompleted/totalWindows.
+        const val PROGRESS_WINDOWS_COMPLETED = "windowsCompleted"
+        const val PROGRESS_TOTAL_WINDOWS = "totalWindows"
+
+        private const val NOTIFICATION_CHANNEL_ID = "light-backup"
+        private const val NOTIFICATION_ID = 1
+    }
+}
 
 // use a worker factory:
 //   class App : Application(), Configuration.Provider {

library/src/jvmSharedMain/kotlin/com/thelightphone/backup/BackupResult.kt

@@ -13,6 +13,9 @@ sealed class FailureScope {
 
 data class BackupFailure(val scope: FailureScope, val cause: Throwable)
 
+// Where "window" is per app, per day
+data class BackupProgress(val windowsCompleted: Int, val totalWindows: Int)
+
 sealed class BackupResult {
     // Every planned file was uploaded.
     data class Completed(val filesBackedUp: Int) : BackupResult()