From 9b0af5be1356f5e04bed2981c5e2f72cf6cc62fa Mon Sep 17 00:00:00 2001 From: Marinov Date: Tue, 20 Oct 2026 00:28:54 +0300 Subject: [PATCH 1/2] Add ocrKeepHighResDetail flag to OCR on full-resolution captures With ocrHighRes enabled, each frame is a full-quality still capture that is then scaled down to the preview size before the scan area is cropped, which discards the detail the capture was taken for. On a Galaxy J6 the Fayda FAN ended up 7-9px tall, below what ML Kit reads reliably. The new experimental flag (default false, no behaviour change) skips the downscale so FrameCropToTargetUseCase crops the scan area from the native-resolution capture. In testing this made the FAN ~40-60px tall and OCR matched 6/7 frames instead of 3/8, at the cost of slower OCR per frame. Co-Authored-By: Claude Opus 5.5 --- .../ExternalCredentialScanOcrFragment.kt | 1 + .../ExternalCredentialScanOcrViewModel.kt | 1 + .../screens/scanocr/model/OcrConfig.kt | 1 + .../infra/camera/CameraFrameProvider.kt | 9 +++++++++ .../NormalizeHighResBitmapToPreviewUseCase.kt | 4 ++++ .../FrameCropToTargetUseCaseTest.kt | 16 ++++++++++++++++ ...rmalizeHighResBitmapToPreviewUseCaseTest.kt | 18 ++++++++++++++++++ .../models/ExperimentalProjectConfiguration.kt | 9 +++++++++ .../ExperimentalProjectConfigurationTest.kt | 12 ++++++++++++ 9 files changed, 71 insertions(+) diff --git a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrFragment.kt b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrFragment.kt index 0f87ca2e2f..948e4095a7 100644 --- a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrFragment.kt +++ b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrFragment.kt @@ -195,6 +195,7 @@ internal class ExternalCredentialScanOcrFragment : Fragment(R.layout.fragment_ex cameraPreviewView = binding.preview, target = targetRect, highResolution = ocrConfig.useHighRes, + keepHighResolutionDetail = ocrConfig.keepHighResDetail, ) { Simber.e("Camera binding failed in OCR", it, MULTI_FACTOR_ID) } diff --git a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModel.kt b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModel.kt index e1f3b7f701..79b935c547 100644 --- a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModel.kt +++ b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModel.kt @@ -87,6 +87,7 @@ internal class ExternalCredentialScanOcrViewModel @AssistedInject constructor( with(configRepository.getProjectConfiguration().experimental()) { ocrConfigFlow.value = OcrConfig( useHighRes = ocrUseHighRes, + keepHighResDetail = ocrKeepHighResDetail, capturesRequired = ocrCaptures.coerceIn(OCR_CAPTURE_MIN, OCR_CAPTURE_MAX), ) } diff --git a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrConfig.kt b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrConfig.kt index 314b03df9a..8c595f33b8 100644 --- a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrConfig.kt +++ b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrConfig.kt @@ -5,5 +5,6 @@ import com.simprints.core.ExcludedFromGeneratedTestCoverageReports @ExcludedFromGeneratedTestCoverageReports("Data struct") internal data class OcrConfig( val useHighRes: Boolean, + val keepHighResDetail: Boolean, val capturesRequired: Int, ) diff --git a/infra/camera/src/main/java/com/simprints/infra/camera/CameraFrameProvider.kt b/infra/camera/src/main/java/com/simprints/infra/camera/CameraFrameProvider.kt index e3ebbb9614..ef2bd67506 100644 --- a/infra/camera/src/main/java/com/simprints/infra/camera/CameraFrameProvider.kt +++ b/infra/camera/src/main/java/com/simprints/infra/camera/CameraFrameProvider.kt @@ -70,6 +70,11 @@ class CameraFrameProvider @Inject internal constructor( private val frameEmissionHelper = FrameEmissionHelper() + // When true, high-resolution captures are not scaled down to the preview size. + // Set on the background dispatcher during initialisation and read on the camera executor. + @Volatile + private var keepHighResolutionDetail = false + fun isInitialised() = camera != null /** @@ -88,8 +93,10 @@ class CameraFrameProvider @Inject internal constructor( cameraPreviewView: CameraPreviewView, target: Rect? = null, highResolution: Boolean = false, + keepHighResolutionDetail: Boolean = false, onError: (Throwable) -> Unit = {}, ) = withContext(bgDispatcher) { + this@CameraFrameProvider.keepHighResolutionDetail = keepHighResolutionDetail val previewView = cameraPreviewView.previewView try { previewRect = fullPreviewSizeRect(previewView) @@ -216,6 +223,7 @@ class CameraFrameProvider @Inject internal constructor( imageCapture = null camera = null frameEmissionHelper.reset() + keepHighResolutionDetail = false if (!executor.isShutdown) executor.shutdown() } @@ -279,6 +287,7 @@ class CameraFrameProvider @Inject internal constructor( rotationDegrees, previewViewWidth, previewViewHeight, + scaleToPreview = !keepHighResolutionDetail, ), rotationDegrees, ) diff --git a/infra/camera/src/main/java/com/simprints/infra/camera/usecase/NormalizeHighResBitmapToPreviewUseCase.kt b/infra/camera/src/main/java/com/simprints/infra/camera/usecase/NormalizeHighResBitmapToPreviewUseCase.kt index 9624558bdb..d7d752cd4a 100644 --- a/infra/camera/src/main/java/com/simprints/infra/camera/usecase/NormalizeHighResBitmapToPreviewUseCase.kt +++ b/infra/camera/src/main/java/com/simprints/infra/camera/usecase/NormalizeHighResBitmapToPreviewUseCase.kt @@ -18,6 +18,8 @@ internal class NormalizeHighResBitmapToPreviewUseCase @Inject constructor() { * sees in the camera preview, making the target bounds spatially consistent with the preview overlay. * * @param originalBitmap the original camera capture bitmap + * @param scaleToPreview when false, step 3 is skipped and the bitmap keeps the capture's native resolution (only the aspect + * ratio matches the PreviewView). Target bounds are still mapped correctly by FrameCropToTargetUseCase. * * @return a new bitmap with normalized dimensions and aspect ratio */ @@ -26,6 +28,7 @@ internal class NormalizeHighResBitmapToPreviewUseCase @Inject constructor() { rotationDegrees: Int, previewWidth: Int, previewHeight: Int, + scaleToPreview: Boolean = true, ): Bitmap { // Rotate if necessary val rotated = if (rotationDegrees != 0) { @@ -57,6 +60,7 @@ internal class NormalizeHighResBitmapToPreviewUseCase @Inject constructor() { } val cropped = Bitmap.createBitmap(rotated, offsetX, offsetY, cropWidth, cropHeight) + if (!scaleToPreview) return cropped // Scale to PreviewView size return cropped.scale(previewWidth, previewHeight) diff --git a/infra/camera/src/test/java/com/simprints/infra/camera/postprocess/FrameCropToTargetUseCaseTest.kt b/infra/camera/src/test/java/com/simprints/infra/camera/postprocess/FrameCropToTargetUseCaseTest.kt index 8ad0a7879f..7ca5b92502 100644 --- a/infra/camera/src/test/java/com/simprints/infra/camera/postprocess/FrameCropToTargetUseCaseTest.kt +++ b/infra/camera/src/test/java/com/simprints/infra/camera/postprocess/FrameCropToTargetUseCaseTest.kt @@ -114,6 +114,22 @@ internal class FrameCropToTargetUseCaseTest { assertThat(cropped.height).isEqualTo(600) } + @Test + fun `Crops at native resolution when high-res bitmap has preview aspect ratio`() { + val cropped = useCase( + Frame( + bitmap = Bitmap.createBitmap(2160, 4062, Bitmap.Config.ARGB_8888), // 3x preview size + rotation = 0, + previewBounds = Rect(0, 0, 720, 1354), + targetBounds = Rect(28, 470, 692, 885), // 664x415 + ), + ) + + // Cropped should keep the 3x detail of the high-res capture + assertThat(cropped.width).isEqualTo(1992) + assertThat(cropped.height).isEqualTo(1245) + } + @Test fun `Correctly crops bitmap with valid cutout rectangle`() { val cropped = useCase( diff --git a/infra/camera/src/test/java/com/simprints/infra/camera/usecase/NormalizeHighResBitmapToPreviewUseCaseTest.kt b/infra/camera/src/test/java/com/simprints/infra/camera/usecase/NormalizeHighResBitmapToPreviewUseCaseTest.kt index 11805f0411..55a1e8e25e 100644 --- a/infra/camera/src/test/java/com/simprints/infra/camera/usecase/NormalizeHighResBitmapToPreviewUseCaseTest.kt +++ b/infra/camera/src/test/java/com/simprints/infra/camera/usecase/NormalizeHighResBitmapToPreviewUseCaseTest.kt @@ -97,4 +97,22 @@ internal class NormalizeHighResBitmapToPreviewUseCaseTest { verify { Bitmap.createBitmap(inputBitmap, 0, 0, inputWidth, inputHeight, any(), true) } verify { croppedBitmap.scale(previewWidth, previewHeight) } } + + @Test + fun `keeps native resolution when scaling to preview is disabled`() = runTest { + every { + Bitmap.createBitmap(inputBitmap, 0, 0, inputWidth, inputHeight, any(), true) + } returns rotatedBitmap + + every { + Bitmap.createBitmap(rotatedBitmap, any(), any(), any(), any()) + } returns croppedBitmap + + val result = useCase(inputBitmap, rotationDegrees, previewWidth, previewHeight, scaleToPreview = false) + + Truth.assertThat(result).isEqualTo(croppedBitmap) + // Rotated 1080x1920 cropped to 4:3 at native resolution: 1080x810, centered vertically + verify { Bitmap.createBitmap(rotatedBitmap, 0, 555, 1080, 810) } + verify(exactly = 0) { croppedBitmap.scale(any(), any()) } + } } diff --git a/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt b/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt index a832cff387..7cec2cd252 100644 --- a/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt +++ b/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt @@ -59,6 +59,13 @@ data class ExperimentalProjectConfiguration( ?.booleanOrNull ?: OCR_USE_HIGH_RES_DEFAULT + val ocrKeepHighResDetail: Boolean + get() = customConfig + ?.get(OCR_KEEP_HIGH_RES_DETAIL) + ?.jsonPrimitive + ?.booleanOrNull + ?: OCR_KEEP_HIGH_RES_DETAIL_DEFAULT + val ocrCaptures: Int get() = customConfig ?.get(OCR_CAPTURES) @@ -252,6 +259,8 @@ data class ExperimentalProjectConfiguration( internal const val OCR_USE_HIGH_RES = "ocrHighRes" internal const val OCR_USE_HIGH_RES_DEFAULT = true + internal const val OCR_KEEP_HIGH_RES_DETAIL = "ocrKeepHighResDetail" + internal const val OCR_KEEP_HIGH_RES_DETAIL_DEFAULT = false internal const val OCR_CAPTURES = "ocrCaptures" internal const val OCR_CAPTURES_DEFAULT = 3 diff --git a/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt b/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt index e95818ef80..6847531e1d 100644 --- a/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt +++ b/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt @@ -137,6 +137,18 @@ internal class ExperimentalProjectConfigurationTest { } } + @Test + fun `check ocr keep high res detail flag correctly`() { + mapOf( + emptyMap() to false, + mapOf(ExperimentalProjectConfiguration.OCR_KEEP_HIGH_RES_DETAIL to JsonPrimitive(1)) to false, + mapOf(ExperimentalProjectConfiguration.OCR_KEEP_HIGH_RES_DETAIL to JsonPrimitive(false)) to false, + mapOf(ExperimentalProjectConfiguration.OCR_KEEP_HIGH_RES_DETAIL to JsonPrimitive(true)) to true, + ).forEach { (config, result) -> + assertThat(ExperimentalProjectConfiguration(config).ocrKeepHighResDetail).isEqualTo(result) + } + } + @Test fun `check ocr captures value correctly`() { val expectedOcrCaptures = 10 From 7757c50050ea095d8454ddbe7a85f5ca63e21789 Mon Sep 17 00:00:00 2001 From: Marinov Date: Tue, 20 Oct 2026 00:30:34 +0300 Subject: [PATCH 2/2] Add ocrReadFaydaBarcode flag to read the Fayda FAN from the card barcode Fayda cards carry the FAN in a Code 128 barcode below the printed number. Barcode decoding is checksummed, so unlike OCR it does not misread digits, and it works at distances where the printed FAN is too small for OCR. When the new experimental flag is enabled (default false, no behaviour change), Fayda scans try the barcode first and only fall back to OCR when no barcode with a valid FAN is found or barcode detection fails. When any capture came from a barcode, the most frequent barcode value is used as the final FAN instead of per-digit voting, so OCR misreads cannot outvote it and differing barcode reads cannot be merged into a value that was never decoded. NHIS and Ghana ID scans are not affected. In testing on a Galaxy J6 this brought a full capture down to ~10-20s with no misread FANs (from ~48s with one misread). Co-Authored-By: Claude Opus 5.5 --- feature/external-credential/build.gradle.kts | 1 + .../ExternalCredentialScanOcrViewModel.kt | 9 +++- .../screens/scanocr/model/OcrConfig.kt | 1 + .../screens/scanocr/model/OcrScanResult.kt | 1 + .../screens/scanocr/reader/ScannedBarcode.kt | 16 ++++++ .../usecase/BuildMfidDocumentUseCase.kt | 14 +++-- .../usecase/FaydaCardOcrReaderUseCase.kt | 30 +++++++++++ .../usecase/ReadBarcodesFromImageUseCase.kt | 45 ++++++++++++++++ .../usecase/ScanMfidDocumentUseCase.kt | 9 ++++ .../ExternalCredentialScanOcrViewModelTest.kt | 34 ++++++++++-- .../usecase/BuildMfidDocumentUseCaseTest.kt | 38 ++++++++++++- .../usecase/FaydaCardOcrReaderUseCaseTest.kt | 47 ++++++++++++++++ .../usecase/ScanMfidDocumentUseCaseTest.kt | 54 +++++++++++++++++++ .../ExperimentalProjectConfiguration.kt | 9 ++++ .../ExperimentalProjectConfigurationTest.kt | 12 +++++ 15 files changed, 311 insertions(+), 9 deletions(-) create mode 100644 feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/reader/ScannedBarcode.kt create mode 100644 feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/ReadBarcodesFromImageUseCase.kt diff --git a/feature/external-credential/build.gradle.kts b/feature/external-credential/build.gradle.kts index 510eb7728e..1dad11563d 100644 --- a/feature/external-credential/build.gradle.kts +++ b/feature/external-credential/build.gradle.kts @@ -20,4 +20,5 @@ dependencies { implementation(project(":infra:camera")) implementation(libs.mlkit.text.recognition) + implementation(libs.playServices.barcode) } diff --git a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModel.kt b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModel.kt index 79b935c547..e594ce8396 100644 --- a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModel.kt +++ b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModel.kt @@ -89,6 +89,7 @@ internal class ExternalCredentialScanOcrViewModel @AssistedInject constructor( useHighRes = ocrUseHighRes, keepHighResDetail = ocrKeepHighResDetail, capturesRequired = ocrCaptures.coerceIn(OCR_CAPTURE_MIN, OCR_CAPTURE_MAX), + readFaydaBarcode = ocrReadFaydaBarcode, ) } lightingConditionsAssessmentConfig = getLightingConditionsAssessmentConfig() @@ -142,8 +143,12 @@ internal class ExternalCredentialScanOcrViewModel @AssistedInject constructor( if (!isOcrAllowed) return@launch val mfidConfig = configRepository.getProjectConfiguration().multifactorId ?: return@launch val captureConfig = ocrConfigFlow.value ?: return@launch - val scannedMfidDocument = - scanMfidDocumentUseCase(bitmap = cropped, documentType = ocrDocumentType, config = mfidConfig) ?: return@launch + val scannedMfidDocument = scanMfidDocumentUseCase( + bitmap = cropped, + documentType = ocrDocumentType, + config = mfidConfig, + readFaydaBarcode = captureConfig.readFaydaBarcode, + ) ?: return@launch Simber.d("Detected OCR") val notEnoughCaptures = scannedMfidDocuments.size < captureConfig.capturesRequired if (isScanningInProgress && notEnoughCaptures) { diff --git a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrConfig.kt b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrConfig.kt index 8c595f33b8..b6c9e0dd4d 100644 --- a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrConfig.kt +++ b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrConfig.kt @@ -7,4 +7,5 @@ internal data class OcrConfig( val useHighRes: Boolean, val keepHighResDetail: Boolean, val capturesRequired: Int, + val readFaydaBarcode: Boolean, ) diff --git a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrScanResult.kt b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrScanResult.kt index 99fc3d7841..0e684c0cad 100644 --- a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrScanResult.kt +++ b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/model/OcrScanResult.kt @@ -41,5 +41,6 @@ internal sealed class OcrScanResult { @SerialName("OcrScanResult.FaydaCard") data class FaydaCard( override val credential: OcrLine, + val isFromBarcode: Boolean = false, ) : OcrScanResult() } diff --git a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/reader/ScannedBarcode.kt b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/reader/ScannedBarcode.kt new file mode 100644 index 0000000000..c9236f37fc --- /dev/null +++ b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/reader/ScannedBarcode.kt @@ -0,0 +1,16 @@ +package com.simprints.feature.externalcredential.screens.scanocr.reader + +import com.simprints.core.ExcludedFromGeneratedTestCoverageReports +import com.simprints.feature.externalcredential.model.BoundingBox + +/** + * A single barcode detected in the scanned image. + * + * @param value decoded barcode value + * @param boundingBox coordinates of the barcode + */ +@ExcludedFromGeneratedTestCoverageReports("Data class") +internal data class ScannedBarcode( + val value: String, + val boundingBox: BoundingBox, +) diff --git a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/BuildMfidDocumentUseCase.kt b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/BuildMfidDocumentUseCase.kt index 38e00f4bfa..163c712f52 100644 --- a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/BuildMfidDocumentUseCase.kt +++ b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/BuildMfidDocumentUseCase.kt @@ -70,11 +70,19 @@ internal class BuildMfidDocumentUseCase @Inject constructor( } private fun buildFaydaCard(scannedDocuments: List): MfidDocument.FaydaCard { + // Barcode readouts are checksummed, so they are preferred over OCR readouts that may contain misread digits. The most frequent + // complete barcode value is used, as per-character voting across differing barcode values could produce a value never decoded. val credential = scannedDocuments + .filter { (it.ocrScanResult as? OcrScanResult.FaydaCard)?.isFromBarcode == true } .map { it.ocrScanResult.credential.text } - .let { credentials -> getBestReadoutBasedOnConfidenceUseCase(credentials, targetLength = CREDENTIAL_LENGTH_FAYDA_CARD) } - .asTokenizableRaw() - return MfidDocument.FaydaCard(credential = credential) + .groupingBy { it } + .eachCount() + .maxByOrNull { it.value } + ?.key + ?: scannedDocuments + .map { it.ocrScanResult.credential.text } + .let { credentials -> getBestReadoutBasedOnConfidenceUseCase(credentials, targetLength = CREDENTIAL_LENGTH_FAYDA_CARD) } + return MfidDocument.FaydaCard(credential = credential.asTokenizableRaw()) } private fun List.bestReadoutOrNull(ocrLine: (T) -> OcrLine?) = mapNotNull(ocrLine) diff --git a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/FaydaCardOcrReaderUseCase.kt b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/FaydaCardOcrReaderUseCase.kt index 9a41260917..5db1d2df0d 100644 --- a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/FaydaCardOcrReaderUseCase.kt +++ b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/FaydaCardOcrReaderUseCase.kt @@ -1,7 +1,10 @@ package com.simprints.feature.externalcredential.screens.scanocr.usecase +import com.simprints.feature.externalcredential.model.BoundingBox import com.simprints.feature.externalcredential.screens.scanocr.model.OcrScanResult +import com.simprints.feature.externalcredential.screens.scanocr.reader.OcrLine import com.simprints.feature.externalcredential.screens.scanocr.reader.OcrReader +import com.simprints.feature.externalcredential.screens.scanocr.reader.ScannedBarcode import javax.inject.Inject internal class FaydaCardOcrReaderUseCase @Inject constructor() { @@ -11,6 +14,31 @@ internal class FaydaCardOcrReaderUseCase @Inject constructor() { } } + /** + * Reads the FAN from the barcode printed below it on the card. Barcode decoding is checksummed, so unlike OCR it does not + * misread individual digits, and it works at distances where the printed FAN is too small for OCR. + */ + fun fromBarcodes(barcodes: List): OcrScanResult.FaydaCard? { + val barcode = barcodes.firstOrNull { FAN_PATTERN.matches(it.value) } ?: return null + // The printed FAN sits directly above the barcode. Expanding the box upwards so that the zoomed-in preview shown to the + // user includes the human-readable number as well. + val box = barcode.boundingBox + val expandedBox = BoundingBox( + left = box.left, + top = box.top - (box.bottom - box.top) / 2, + right = box.right, + bottom = box.bottom, + ) + val credential = OcrLine( + id = BARCODE_LINE_ID, + text = barcode.value, + boundingBox = expandedBox, + blockBoundingBox = expandedBox, + confidence = 1f, + ) + return OcrScanResult.FaydaCard(credential, isFromBarcode = true) + } + companion object { // Fayda Alias Number (FAN): exactly 16 digits val FAN_PATTERN = Regex("^\\d{16}$") @@ -20,5 +48,7 @@ internal class FaydaCardOcrReaderUseCase @Inject constructor() { // Punctuation between digits and more than 4 groups are not allowed, so that adjacent dates such as // "07/02/1998 | 2005/10/17" are not read as a FAN, even if OCR reads the separators as spaces ("07 02 1998 2005 10 17"). val FAN_OCR_PATTERN = Regex("(? = try { + val image = InputImage.fromBitmap(bitmap, 0) + Tasks.await(scanner.process(image)).orEmpty().mapNotNull { barcode -> + val value = barcode.rawValue ?: return@mapNotNull null + val box = barcode.boundingBox ?: return@mapNotNull null + ScannedBarcode(value = value, boundingBox = box.toBoundingBox()) + } + } catch (e: Exception) { + Simber.i("Barcode detection failed", e, tag = MULTI_FACTOR_ID) + emptyList() + } +} diff --git a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/ScanMfidDocumentUseCase.kt b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/ScanMfidDocumentUseCase.kt index cd874bac94..07cf9f9d18 100644 --- a/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/ScanMfidDocumentUseCase.kt +++ b/feature/external-credential/src/main/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/ScanMfidDocumentUseCase.kt @@ -15,6 +15,7 @@ import javax.inject.Singleton @Singleton internal class ScanMfidDocumentUseCase @Inject constructor( private val readTextFromImage: ReadTextFromImageUseCase, + private val readBarcodesFromImage: ReadBarcodesFromImageUseCase, private val ghanaNhisCardOcrReaderUseCase: GhanaNhisCardOcrReaderUseCase, private val ghanaIdCardOcrReaderUseCase: GhanaIdCardOcrReaderUseCase, private val faydaCardOcrReaderUseCase: FaydaCardOcrReaderUseCase, @@ -24,6 +25,7 @@ internal class ScanMfidDocumentUseCase @Inject constructor( bitmap: Bitmap, documentType: OcrDocumentType, config: MultiFactorIdConfiguration, + readFaydaBarcode: Boolean = false, ): ScannedMfidDocument? { val isCapturingAllFields = when (documentType) { OcrDocumentType.NhisCard -> config.nhisCardConfig?.isCapturingAllFields @@ -31,6 +33,13 @@ internal class ScanMfidDocumentUseCase @Inject constructor( OcrDocumentType.FaydaCard -> config.faydaCardConfig?.isCapturingAllFields } ?: false return try { + if (readFaydaBarcode && documentType == OcrDocumentType.FaydaCard) { + // Fayda cards carry the FAN in a barcode, which reads faster and more reliably than the printed number + faydaCardOcrReaderUseCase.fromBarcodes(readBarcodesFromImage(bitmap))?.let { scanResult -> + val savedImagePath = credentialImageRepository.saveCredentialScan(bitmap, imageType = FullDocument) + return ScannedMfidDocument(savedImagePath, scanResult) + } + } val ocrText = readTextFromImage(bitmap) ?: return null val ocrReader = OcrReader(ocrText) val scanResult = when (documentType) { diff --git a/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModelTest.kt b/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModelTest.kt index 1532bbff89..7ef56f261b 100644 --- a/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModelTest.kt +++ b/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/ExternalCredentialScanOcrViewModelTest.kt @@ -18,6 +18,7 @@ import com.simprints.infra.camera.Frame import com.simprints.infra.camera.postprocess.FrameCropToTargetUseCase import com.simprints.infra.config.store.ConfigRepository import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.MFID_LIGHTING_CONDITIONS_ASSESSMENT_ENABLED +import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.OCR_READ_FAYDA_BARCODE import com.simprints.infra.config.store.models.ProjectConfiguration import com.simprints.infra.resources.R import com.simprints.testtools.common.coroutines.TestCoroutineRule @@ -134,6 +135,33 @@ internal class ExternalCredentialScanOcrViewModelTest { assertThat(viewModel.isOcrActive).isTrue() } + @Test + fun `processImage passes Fayda barcode flag from config to scan use case`() = runTest { + val mockCroppedBitmap = mockk() + coEvery { frameCropToTargetUseCase(frame) } returns mockCroppedBitmap + viewModel = initViewModel( + documentType = OcrDocumentType.FaydaCard, + customConfig = mapOf(OCR_READ_FAYDA_BARCODE to JsonPrimitive(true)), + ) + + viewModel.startScanning() + viewModel.processImage(frame) + + coVerify { scanMfidDocumentUseCase(mockCroppedBitmap, OcrDocumentType.FaydaCard, any(), readFaydaBarcode = true) } + } + + @Test + fun `processImage does not read Fayda barcode by default`() = runTest { + val mockCroppedBitmap = mockk() + coEvery { frameCropToTargetUseCase(frame) } returns mockCroppedBitmap + viewModel = initViewModel(documentType = OcrDocumentType.FaydaCard) + + viewModel.startScanning() + viewModel.processImage(frame) + + coVerify { scanMfidDocumentUseCase(mockCroppedBitmap, OcrDocumentType.FaydaCard, any(), readFaydaBarcode = false) } + } + @Test fun `processImage updates lighting conditions after debounce`() = runTest { val mockCroppedBitmap = mockk() @@ -171,7 +199,7 @@ internal class ExternalCredentialScanOcrViewModelTest { assertThat(observer.value()).isInstanceOf(ScanOcrState.NotScanning::class.java) coVerify(exactly = 0) { frameCropToTargetUseCase(frame) } - coVerify(exactly = 0) { scanMfidDocumentUseCase.invoke(any(), any(), any()) } + coVerify(exactly = 0) { scanMfidDocumentUseCase.invoke(any(), any(), any(), any()) } coVerify(exactly = 0) { getLightingConditionsAssessment.invoke(any(), any()) } assertThat(viewModel.isOcrActive).isFalse() } @@ -200,7 +228,7 @@ internal class ExternalCredentialScanOcrViewModelTest { assertThat(observer.value()).isEqualTo(lightingConditionsAssessment) coVerify(exactly = 1) { getLightingConditionsAssessment(mockCroppedBitmap, any()) } - coVerify(exactly = 0) { scanMfidDocumentUseCase.invoke(any(), any(), any()) } + coVerify(exactly = 0) { scanMfidDocumentUseCase.invoke(any(), any(), any(), any()) } } @Test @@ -377,6 +405,6 @@ internal class ExternalCredentialScanOcrViewModelTest { assertThat(stateObserver.value()).isEqualTo(ScanOcrState.Complete) assertThat(viewModel.isOcrActive).isFalse() - coVerify(exactly = 0) { scanMfidDocumentUseCase.invoke(any(), any(), any()) } + coVerify(exactly = 0) { scanMfidDocumentUseCase.invoke(any(), any(), any(), any()) } } } diff --git a/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/BuildMfidDocumentUseCaseTest.kt b/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/BuildMfidDocumentUseCaseTest.kt index 589fc33a95..33bba9b713 100644 --- a/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/BuildMfidDocumentUseCaseTest.kt +++ b/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/BuildMfidDocumentUseCaseTest.kt @@ -187,6 +187,38 @@ internal class BuildMfidDocumentUseCaseTest { verify { getBestReadout(listOf(credentialText), targetLength = FAYDA_CREDENTIAL_LENGTH) } } + @Test + fun `uses only barcode readouts for FaydaCard when available`() { + val documents = listOf( + faydaScannedDocument(credential = "8642370540168184"), + faydaScannedDocument(credential = "8642370540168134", isFromBarcode = true), + faydaScannedDocument(credential = "8642370540168184"), + ) + val result = useCase(documents, OcrDocumentType.FaydaCard) as MfidDocument.FaydaCard + assertThat(result.credential).isEqualTo("8642370540168134".asTokenizableRaw()) + verify(exactly = 0) { getBestReadout(any(), any()) } + } + + @Test + fun `uses most frequent complete barcode value for FaydaCard`() { + val documents = listOf( + faydaScannedDocument(credential = "2580945924815610", isFromBarcode = true), + faydaScannedDocument(credential = "2580945924815010", isFromBarcode = true), + faydaScannedDocument(credential = "2580945924815610", isFromBarcode = true), + ) + val result = useCase(documents, OcrDocumentType.FaydaCard) as MfidDocument.FaydaCard + assertThat(result.credential).isEqualTo("2580945924815610".asTokenizableRaw()) + } + + @Test + fun `returns one of the decoded barcode values when all barcode readouts differ`() { + val barcodeValues = listOf("2580945924815100", "2580945924815010", "2580945924815001") + val documents = barcodeValues.map { faydaScannedDocument(credential = it, isFromBarcode = true) } + val result = useCase(documents, OcrDocumentType.FaydaCard) as MfidDocument.FaydaCard + assertThat(result.credential).isIn(barcodeValues.map { it.asTokenizableRaw() }) + verify(exactly = 0) { getBestReadout(any(), any()) } + } + @Test fun `sets credential from best readout for FaydaCard`() { val expectedCredential = "1234567812345678" @@ -233,10 +265,14 @@ internal class BuildMfidDocumentUseCaseTest { verify { getBestReadout(texts, targetLength = NHIS_CREDENTIAL_LENGTH) } } - private fun faydaScannedDocument(credential: String = "1234567812345678") = ScannedMfidDocument( + private fun faydaScannedDocument( + credential: String = "1234567812345678", + isFromBarcode: Boolean = false, + ) = ScannedMfidDocument( imagePath = "path/to/image.jpg", ocrScanResult = OcrScanResult.FaydaCard( credential = ocrLine(credential), + isFromBarcode = isFromBarcode, ), ) diff --git a/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/FaydaCardOcrReaderUseCaseTest.kt b/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/FaydaCardOcrReaderUseCaseTest.kt index e3a6e5f1b2..078da6093f 100644 --- a/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/FaydaCardOcrReaderUseCaseTest.kt +++ b/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/FaydaCardOcrReaderUseCaseTest.kt @@ -5,6 +5,7 @@ import com.simprints.feature.externalcredential.model.BoundingBox import com.simprints.feature.externalcredential.screens.scanocr.reader.OcrLine import com.simprints.feature.externalcredential.screens.scanocr.reader.OcrReader import com.simprints.feature.externalcredential.screens.scanocr.reader.OcrText +import com.simprints.feature.externalcredential.screens.scanocr.reader.ScannedBarcode import io.mockk.MockKAnnotations import org.junit.Before import org.junit.Test @@ -105,6 +106,52 @@ internal class FaydaCardOcrReaderUseCaseTest { assertThat(result?.credential?.text).isEqualTo("8642370540168134") } + @Test + fun `returns FAN from barcode`() { + val barcodes = listOf(barcode("2580945924815610", top = 100, bottom = 140)) + + val result = useCase.fromBarcodes(barcodes) + + assertThat(result?.credential?.text).isEqualTo("2580945924815610") + assertThat(result?.credential?.confidence).isEqualTo(1f) + assertThat(result?.isFromBarcode).isTrue() + } + + @Test + fun `expands barcode box upwards to include printed FAN`() { + val result = useCase.fromBarcodes(listOf(barcode("2580945924815610", top = 100, bottom = 140))) + + assertThat(result?.credential?.boundingBox).isEqualTo(BoundingBox(left = 10, top = 80, right = 200, bottom = 140)) + assertThat(result?.credential?.blockBoundingBox).isEqualTo(result?.credential?.boundingBox) + } + + @Test + fun `skips barcodes that are not a FAN`() { + val barcodes = listOf( + barcode("ABC123", top = 0, bottom = 10), + barcode("123456781234567", top = 0, bottom = 10), // 15 digits + barcode("1234 5678 1234 5678", top = 0, bottom = 10), // separators are not expected in barcodes + barcode("2580945924815610", top = 100, bottom = 140), + ) + + assertThat(useCase.fromBarcodes(barcodes)?.credential?.text).isEqualTo("2580945924815610") + } + + @Test + fun `returns null when no barcode is a FAN`() { + assertThat(useCase.fromBarcodes(emptyList())).isNull() + assertThat(useCase.fromBarcodes(listOf(barcode("12345678123456789", top = 0, bottom = 10)))).isNull() + } + + private fun barcode( + value: String, + top: Int, + bottom: Int, + ) = ScannedBarcode( + value = value, + boundingBox = BoundingBox(left = 10, top = top, right = 200, bottom = bottom), + ) + private fun buildReader(vararg lines: OcrLine) = OcrReader( OcrText(allLines = lines.toList()), ) diff --git a/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/ScanMfidDocumentUseCaseTest.kt b/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/ScanMfidDocumentUseCaseTest.kt index 50c49e1f28..2c4857ffbc 100644 --- a/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/ScanMfidDocumentUseCaseTest.kt +++ b/feature/external-credential/src/test/java/com/simprints/feature/externalcredential/screens/scanocr/usecase/ScanMfidDocumentUseCaseTest.kt @@ -6,6 +6,7 @@ import com.simprints.feature.externalcredential.model.BoundingBox import com.simprints.feature.externalcredential.screens.scanocr.model.OcrDocumentType import com.simprints.feature.externalcredential.screens.scanocr.model.OcrScanResult import com.simprints.feature.externalcredential.screens.scanocr.reader.OcrLine +import com.simprints.feature.externalcredential.screens.scanocr.reader.ScannedBarcode import com.simprints.infra.config.store.models.FaydaCardConfig import com.simprints.infra.config.store.models.GhanaIdCardConfig import com.simprints.infra.config.store.models.MultiFactorIdConfiguration @@ -26,6 +27,9 @@ internal class ScanMfidDocumentUseCaseTest { @MockK private lateinit var readTextFromImage: ReadTextFromImageUseCase + @MockK + private lateinit var readBarcodesFromImage: ReadBarcodesFromImageUseCase + @MockK private lateinit var ghanaNhisCardOcrReaderUseCase: GhanaNhisCardOcrReaderUseCase @@ -76,12 +80,15 @@ internal class ScanMfidDocumentUseCaseTest { MockKAnnotations.init(this, relaxed = true) useCase = ScanMfidDocumentUseCase( readTextFromImage = readTextFromImage, + readBarcodesFromImage = readBarcodesFromImage, ghanaNhisCardOcrReaderUseCase = ghanaNhisCardOcrReaderUseCase, ghanaIdCardOcrReaderUseCase = ghanaIdCardOcrReaderUseCase, faydaCardOcrReaderUseCase = faydaCardOcrReaderUseCase, credentialImageRepository = credentialImageRepository, ) coEvery { credentialImageRepository.saveCredentialScan(any(), any()) } returns savedImagePath + every { readBarcodesFromImage(any()) } returns emptyList() + every { faydaCardOcrReaderUseCase.fromBarcodes(any()) } returns null } @Test @@ -164,6 +171,53 @@ internal class ScanMfidDocumentUseCaseTest { verify(exactly = 0) { ghanaIdCardOcrReaderUseCase(any(), any()) } } + @Test + fun `returns FaydaCard from barcode without running OCR`() = runTest { + val barcodes = listOf(ScannedBarcode(value = "1234567812345678", boundingBox = BoundingBox(0, 0, 10, 10))) + val barcodeResult = OcrScanResult.FaydaCard(credential = ocrLine(text = "1234567812345678")) + every { readBarcodesFromImage(bitmap) } returns barcodes + every { faydaCardOcrReaderUseCase.fromBarcodes(barcodes) } returns barcodeResult + + val result = useCase(bitmap, OcrDocumentType.FaydaCard, faydaConfig, readFaydaBarcode = true) + + assertThat(result?.ocrScanResult).isEqualTo(barcodeResult) + verify(exactly = 0) { readTextFromImage(any()) } + verify(exactly = 0) { faydaCardOcrReaderUseCase(any()) } + } + + @Test + fun `falls back to OCR for FaydaCard when barcode is not read`() = runTest { + val ocrResult = OcrScanResult.FaydaCard(credential = ocrLine(text = "1234567812345678")) + every { readTextFromImage(bitmap) } returns mockk(relaxed = true) + every { faydaCardOcrReaderUseCase(any()) } returns ocrResult + + val result = useCase(bitmap, OcrDocumentType.FaydaCard, faydaConfig, readFaydaBarcode = true) + + assertThat(result?.ocrScanResult).isEqualTo(ocrResult) + verify(exactly = 1) { readBarcodesFromImage(bitmap) } + verify(exactly = 1) { readTextFromImage(bitmap) } + } + + @Test + fun `does not read barcodes for Ghana cards`() = runTest { + every { readTextFromImage(bitmap) } returns mockk(relaxed = true) + + useCase(bitmap, OcrDocumentType.NhisCard, nhisConfig, readFaydaBarcode = true) + useCase(bitmap, OcrDocumentType.GhanaIdCard, ghanaIdConfig, readFaydaBarcode = true) + + verify(exactly = 0) { readBarcodesFromImage(any()) } + } + + @Test + fun `does not read barcodes for FaydaCard when disabled`() = runTest { + every { readTextFromImage(bitmap) } returns mockk(relaxed = true) + + useCase(bitmap, OcrDocumentType.FaydaCard, faydaConfig, readFaydaBarcode = false) + + verify(exactly = 0) { readBarcodesFromImage(any()) } + verify(exactly = 1) { faydaCardOcrReaderUseCase(any()) } + } + @Test fun `saves image when line is found`() = runTest { every { readTextFromImage(bitmap) } returns mockk(relaxed = true) diff --git a/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt b/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt index 7cec2cd252..c0867742cb 100644 --- a/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt +++ b/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt @@ -73,6 +73,13 @@ data class ExperimentalProjectConfiguration( ?.intOrNull ?: OCR_CAPTURES_DEFAULT + val ocrReadFaydaBarcode: Boolean + get() = customConfig + ?.get(OCR_READ_FAYDA_BARCODE) + ?.jsonPrimitive + ?.booleanOrNull + ?: OCR_READ_FAYDA_BARCODE_DEFAULT + val allowConfirmingGuidsNotInCallback: Boolean get() = customConfig ?.get(ALLOW_CONFIRMING_GUIDS_NOT_IN_CALLBACK) @@ -263,6 +270,8 @@ data class ExperimentalProjectConfiguration( internal const val OCR_KEEP_HIGH_RES_DETAIL_DEFAULT = false internal const val OCR_CAPTURES = "ocrCaptures" internal const val OCR_CAPTURES_DEFAULT = 3 + const val OCR_READ_FAYDA_BARCODE = "ocrReadFaydaBarcode" + internal const val OCR_READ_FAYDA_BARCODE_DEFAULT = false const val ALLOW_CONFIRMING_GUIDS_NOT_IN_CALLBACK = "allowConfirmingGuidsNotInCallback" diff --git a/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt b/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt index 6847531e1d..d4bfc24f95 100644 --- a/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt +++ b/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt @@ -162,6 +162,18 @@ internal class ExperimentalProjectConfigurationTest { } } + @Test + fun `check ocr read fayda barcode flag correctly`() { + mapOf( + emptyMap() to false, + mapOf(ExperimentalProjectConfiguration.OCR_READ_FAYDA_BARCODE to JsonPrimitive(1)) to false, + mapOf(ExperimentalProjectConfiguration.OCR_READ_FAYDA_BARCODE to JsonPrimitive(false)) to false, + mapOf(ExperimentalProjectConfiguration.OCR_READ_FAYDA_BARCODE to JsonPrimitive(true)) to true, + ).forEach { (config, result) -> + assertThat(ExperimentalProjectConfiguration(config).ocrReadFaydaBarcode).isEqualTo(result) + } + } + @Test fun `check allow confirming GUIDs not in callback flag correctly`() { mapOf(