diff --git a/ui/src/data-services/models/algorithm.ts b/ui/src/data-services/models/algorithm.ts index c36d507c4..71a854f4d 100644 --- a/ui/src/data-services/models/algorithm.ts +++ b/ui/src/data-services/models/algorithm.ts @@ -3,6 +3,25 @@ import { Entity } from './entity' export type ServerAlgorithm = any // TODO: Update this type +export interface ServerAlgorithmEvaluation { + id: number + occurrence_set: { id: number; name: string } + accuracy: number | null + accuracy_by_species: number | null + occurrences_scored: number + occurrences_skipped: number + species_scored: number + created_at: string +} + +export interface AlgorithmEvaluation { + id: string + occurrenceSetName: string + accuracy?: number + accuracyBySpecies?: number + occurrencesScored: number +} + export class Algorithm extends Entity { protected readonly _algorithm: ServerAlgorithm @@ -43,4 +62,16 @@ export class Algorithm extends Entity { ? this._algorithm.category_count : undefined } + + get evaluations(): AlgorithmEvaluation[] { + const rows: ServerAlgorithmEvaluation[] = this._algorithm.evaluations ?? [] + + return rows.map((row) => ({ + id: `${row.id}`, + occurrenceSetName: row.occurrence_set.name, + accuracy: row.accuracy ?? undefined, + accuracyBySpecies: row.accuracy_by_species ?? undefined, + occurrencesScored: row.occurrences_scored, + })) + } } diff --git a/ui/src/data-services/models/species-details.ts b/ui/src/data-services/models/species-details.ts index 8ade4b46d..d17c78b7e 100644 --- a/ui/src/data-services/models/species-details.ts +++ b/ui/src/data-services/models/species-details.ts @@ -3,11 +3,43 @@ import { ServerSpecies, Species } from './species' export type ServerSpeciesDetails = ServerSpecies & any // TODO: Update this type +export interface ServerAlgorithmPerformance { + algorithm: { id: number; name: string; key: string; version: number } + occurrence_set: { id: number; name: string } + accuracy: number + occurrences_scored: number + correct: number + overall_accuracy: number | null + overall_accuracy_by_species: number | null +} + +export interface AlgorithmPerformance { + algorithmId: string + algorithmName: string + accuracy: number + occurrencesScored: number + correct: number + occurrenceSetName: string +} + export class SpeciesDetails extends Species { public constructor(species: ServerSpeciesDetails) { super(species) } + get algorithmPerformance(): AlgorithmPerformance[] { + const rows = this._species.algorithm_performance ?? [] + + return rows.map((row: ServerAlgorithmPerformance) => ({ + algorithmId: `${row.algorithm.id}`, + algorithmName: row.algorithm.name, + accuracy: row.accuracy, + occurrencesScored: row.occurrences_scored, + correct: row.correct, + occurrenceSetName: row.occurrence_set.name, + })) + } + get commonNameLabel(): string | undefined { return this._species.common_name_en ?? undefined } diff --git a/ui/src/data-services/models/species.ts b/ui/src/data-services/models/species.ts index 37d6d622c..fd220b867 100644 --- a/ui/src/data-services/models/species.ts +++ b/ui/src/data-services/models/species.ts @@ -119,6 +119,10 @@ export class Species extends Taxon { return this._species.occurrences_count ?? 0 } + get numTrainingCrops(): number | undefined { + return this._species.training_crops_ready ?? undefined + } + get numVerified(): number { return this._species.verified_count ?? 0 } diff --git a/ui/src/data-services/models/taxa-list.ts b/ui/src/data-services/models/taxa-list.ts index bafd01781..69d748fc6 100644 --- a/ui/src/data-services/models/taxa-list.ts +++ b/ui/src/data-services/models/taxa-list.ts @@ -1,6 +1,22 @@ import { Entity, ServerEntity } from 'data-services/models/entity' +export type ServerTaxaListBestModel = { + id: number + name: string + accuracy: number | null + accuracy_by_species: number | null + occurrence_set: string +} + +export interface TaxaListBestModel { + id: string + name: string + accuracyBySpecies?: number + occurrenceSetName: string +} + export type ServerTaxaList = ServerEntity & { + best_model: ServerTaxaListBestModel | null // Highest scoring algorithm on these taxa projects: number[] // Array of project IDs taxa: string // URL to taxa API endpoint (filtered by this taxa list) taxa_count: number // Number of taxa in list @@ -15,6 +31,21 @@ export class TaxaList extends Entity { this._taxaList = taxaList } + get bestModel(): TaxaListBestModel | undefined { + const model = this._taxaList.best_model + + if (!model) { + return undefined + } + + return { + id: `${model.id}`, + name: model.name, + accuracyBySpecies: model.accuracy_by_species ?? undefined, + occurrenceSetName: model.occurrence_set, + } + } + get taxaCount() { return this._taxaList.taxa_count } diff --git a/ui/src/data-services/types.ts b/ui/src/data-services/types.ts index 49eec8291..fe40b10f8 100644 --- a/ui/src/data-services/types.ts +++ b/ui/src/data-services/types.ts @@ -5,6 +5,7 @@ export interface FetchParams { filters?: { field: string; value?: string; error?: string }[] withCounts?: boolean withExampleOccurrences?: boolean + withTrainingCropCounts?: boolean } export interface APIValidationError { diff --git a/ui/src/data-services/utils.ts b/ui/src/data-services/utils.ts index 9d017b984..0dc304079 100644 --- a/ui/src/data-services/utils.ts +++ b/ui/src/data-services/utils.ts @@ -38,6 +38,9 @@ export const getFetchUrl = ({ if (params?.withExampleOccurrences) { queryParams.with_example_occurrences = String(true) } + if (params?.withTrainingCropCounts) { + queryParams.with_training_crop_counts = String(true) + } const baseUrl = `${API_URL}/${collection}` const queryString = new URLSearchParams(queryParams).toString() diff --git a/ui/src/pages/algorithm-details/algorithm-details-dialog.tsx b/ui/src/pages/algorithm-details/algorithm-details-dialog.tsx index c96e0cb9c..d7af6af64 100644 --- a/ui/src/pages/algorithm-details/algorithm-details-dialog.tsx +++ b/ui/src/pages/algorithm-details/algorithm-details-dialog.tsx @@ -8,6 +8,7 @@ import { useNavigate, useParams } from 'react-router-dom' import { APP_ROUTES } from 'utils/constants' import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' +import { AlgorithmEvaluations } from './algorithm-evaluations' import styles from './styles.module.scss' export const AlgorithmDetailsDialog = ({ id }: { id: string }) => { @@ -91,6 +92,13 @@ const AlgorithmDetailsContent = ({ algorithm }: { algorithm: Algorithm }) => ( /> + {algorithm.evaluations.length > 0 && ( + +
+ +
+
+ )} {algorithm.uri || algorithm.categoryMapURI ? (
diff --git a/ui/src/pages/algorithm-details/algorithm-evaluations.tsx b/ui/src/pages/algorithm-details/algorithm-evaluations.tsx new file mode 100644 index 000000000..f3a50b0b2 --- /dev/null +++ b/ui/src/pages/algorithm-details/algorithm-evaluations.tsx @@ -0,0 +1,71 @@ +import { Algorithm, AlgorithmEvaluation } from 'data-services/models/algorithm' +import { + BasicTableCell, + Table, + TableBackgroundTheme, + TableColumn, + TextAlign, +} from 'nova-ui-kit' +import { STRING, translate } from 'utils/language' + +const ACCURACY_DECIMALS = 2 + +const accuracyLabel = (accuracy?: number) => + accuracy !== undefined + ? accuracy.toFixed(ACCURACY_DECIMALS) + : translate(STRING.VALUE_NOT_AVAILABLE) + +export const columns: TableColumn[] = [ + { + id: 'occurrence-set', + name: translate(STRING.FIELD_LABEL_EVALUATION_SET), + renderCell: (item: AlgorithmEvaluation) => ( + + ), + }, + { + id: 'accuracy', + name: translate(STRING.FIELD_LABEL_ACCURACY), + styles: { + textAlign: TextAlign.Right, + }, + renderCell: (item: AlgorithmEvaluation) => ( + + ), + }, + { + id: 'accuracy-by-species', + name: translate(STRING.FIELD_LABEL_ACCURACY_BY_SPECIES), + styles: { + textAlign: TextAlign.Right, + }, + renderCell: (item: AlgorithmEvaluation) => ( + + ), + }, + { + id: 'occurrences-scored', + name: translate(STRING.FIELD_LABEL_OCCURRENCES_SCORED), + styles: { + textAlign: TextAlign.Right, + }, + renderCell: (item: AlgorithmEvaluation) => ( + + ), + }, +] + +export const AlgorithmEvaluations = ({ + algorithm, +}: { + algorithm: Algorithm +}) => ( + +) diff --git a/ui/src/pages/species-details/species-details.tsx b/ui/src/pages/species-details/species-details.tsx index 81e8c5f53..7e09b527e 100644 --- a/ui/src/pages/species-details/species-details.tsx +++ b/ui/src/pages/species-details/species-details.tsx @@ -5,7 +5,10 @@ import { TaxonDetails } from 'components/taxon-details/taxon-details' import { Tag } from 'components/taxon-tags/tag' import { TagsForm } from 'components/taxon-tags/tags-form' import { useProjectDetails } from 'data-services/hooks/projects/useProjectDetails' -import { SpeciesDetails as Species } from 'data-services/models/species-details' +import { + AlgorithmPerformance, + SpeciesDetails as Species, +} from 'data-services/models/species-details' import { ExternalLinkIcon, LockIcon } from 'lucide-react' import { Box, @@ -31,6 +34,19 @@ export const TABS = { CHARTS: 'charts', } +const ACCURACY_DECIMALS = 2 + +const performanceLabel = ({ + algorithmName, + accuracy, + correct, + occurrencesScored, + occurrenceSetName, +}: AlgorithmPerformance) => + `${algorithmName}: ${accuracy.toFixed( + ACCURACY_DECIMALS + )} (${correct}/${occurrencesScored}) on ${occurrenceSetName}` + export const SpeciesDetails = ({ species, selectedTab, @@ -195,6 +211,26 @@ export const SpeciesDetails = ({ /> + + {species.algorithmPerformance.length ? ( + species.algorithmPerformance.map((performance) => ( + + )) + ) : ( + + )} + ), }, + { + id: 'training-images-ready', + name: translate(STRING.FIELD_LABEL_TRAINING_IMAGES_READY), + styles: { + textAlign: TextAlign.Right, + }, + renderCell: (item: Species) => ( + + ), + }, { id: 'best-determination-score', name: translate(STRING.FIELD_LABEL_BEST_SCORE), diff --git a/ui/src/pages/species/species.tsx b/ui/src/pages/species/species.tsx index 23c813b90..37ebccd47 100644 --- a/ui/src/pages/species/species.tsx +++ b/ui/src/pages/species/species.tsx @@ -51,6 +51,7 @@ export const Species = () => { 'last-seen': true, occurrences: true, verified: true, + 'training-images-ready': true, 'best-determination-score': true, 'created-at': false, 'updated-at': false, @@ -65,6 +66,7 @@ export const Species = () => { filters, // This list renders the Example column and links Last-seen / Best-score to an occurrence. withExampleOccurrences: true, + withTrainingCropCounts: columnSettings['training-images-ready'], }) // Ordered example occurrences, one per taxon row that has one, so the modal's // prev/next steps to the next taxon's example (rows without an example are skipped). diff --git a/ui/src/pages/taxa-lists/taxa-list-columns.tsx b/ui/src/pages/taxa-lists/taxa-list-columns.tsx index 8b84f4c09..ad11ee26d 100644 --- a/ui/src/pages/taxa-lists/taxa-list-columns.tsx +++ b/ui/src/pages/taxa-lists/taxa-list-columns.tsx @@ -1,5 +1,5 @@ import { API_ROUTES } from 'data-services/constants' -import { TaxaList } from 'data-services/models/taxa-list' +import { TaxaList, TaxaListBestModel } from 'data-services/models/taxa-list' import { BasicTableCell, CellTheme, @@ -16,6 +16,19 @@ import { getAppRoute } from 'utils/getAppRoute' import { STRING, translate } from 'utils/language' import { AddTaxaListTaxonPopover } from '../taxa-list-details/add-taxa-list-taxon/add-taxa-list-taxon-popover' +const ACCURACY_DECIMALS = 2 + +// Lists are ranked on the per-species average, so that is the number worth showing. +const bestModelDetails = (bestModel: TaxaListBestModel) => + [ + bestModel.accuracyBySpecies !== undefined + ? `${translate( + STRING.FIELD_LABEL_ACCURACY_BY_SPECIES + )}: ${bestModel.accuracyBySpecies.toFixed(ACCURACY_DECIMALS)}` + : undefined, + bestModel.occurrenceSetName, + ].filter((detail): detail is string => detail !== undefined) + export const columns = ({ projectId, }: { @@ -76,6 +89,28 @@ export const columns = ({ sortField: 'updated_at', renderCell: (item: TaxaList) => , }, + { + id: 'best-model', + name: translate(STRING.FIELD_LABEL_BEST_MODEL), + renderCell: (item: TaxaList) => + item.bestModel ? ( + + + + ) : ( + + ), + }, { id: 'actions', diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index d429fceec..3c643879a 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -81,9 +81,13 @@ export enum STRING { ENTITY_VIEW, /* FIELD_LABEL */ + FIELD_LABEL_ACCURACY_BY_SPECIES, + FIELD_LABEL_ACCURACY, FIELD_LABEL_ADDED_AT, FIELD_LABEL_ALGORITHM_URI, + FIELD_LABEL_ALGORITHM, FIELD_LABEL_AVG_TEMP, + FIELD_LABEL_BEST_MODEL, FIELD_LABEL_BEST_SCORE, FIELD_LABEL_CAPTURE_SET, FIELD_LABEL_OCCURRENCE_SET, @@ -96,6 +100,7 @@ export enum STRING { FIELD_LABEL_COMMENT, FIELD_LABEL_CONNECTION_STATUS, FIELD_LABEL_CREATED_AT, + FIELD_LABEL_CROPS_WITHOUT_EMBEDDING, FIELD_LABEL_DATA_SOURCE_CAPTURES, FIELD_LABEL_DATA_SOURCE, FIELD_LABEL_DATE_OBSERVED, @@ -110,6 +115,7 @@ export enum STRING { FIELD_LABEL_EMAIL, FIELD_LABEL_ENDPOINT, FIELD_LABEL_ERRORS, + FIELD_LABEL_EVALUATION_SET, FIELD_LABEL_EXAMPLE, FIELD_LABEL_FILE_SIZE, FIELD_LABEL_FILENAME, @@ -140,6 +146,7 @@ export enum STRING { FIELD_LABEL_NEW_PASSWORD, FIELD_LABEL_NUM_PIPELINES_REGISTERED, FIELD_LABEL_OCCURRENCES, + FIELD_LABEL_OCCURRENCES_SCORED, FIELD_LABEL_PASSWORD_CURRENT, FIELD_LABEL_PASSWORD_NEW, FIELD_LABEL_PASSWORD, @@ -176,6 +183,8 @@ export enum STRING { FIELD_LABEL_TOTAL_SIZE, FIELD_LABEL_TRAINING_IMAGES, FIELD_LABEL_TYPE, + FIELD_LABEL_SPECIES, + FIELD_LABEL_TRAINING_IMAGES_READY, FIELD_LABEL_UPDATED_AT, FIELD_LABEL_USER, FIELD_LABEL_VERSION_NAME, @@ -286,6 +295,7 @@ export enum STRING { TOOLTIP_CAPTURE, TOOLTIP_DEPLOYMENT, TOOLTIP_DEVICE, + TOOLTIP_EVALUATION_SET, TOOLTIP_JOB, TOOLTIP_LATEST_JOB_STATUS, TOOLTIP_OCCURRENCE, @@ -310,6 +320,7 @@ export enum STRING { AGREEMENT_EXACT, AGREEMENT_KAPPA, ALGORITHMS, + ALGORITHMS_AND_PERFORMANCE, ANONYMOUS_USER, APPLY_ID_SHORT, APPLY_ID, @@ -321,6 +332,7 @@ export enum STRING { CONNECTING, DEFAULT, DETAILS, + EVALUATION, EXTERNAL_RESOURCES, FORGOT_PASSWORD_DETAILS, FORGOT_PASSWORD, @@ -450,9 +462,13 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.ZOOM_OUT]: 'Zoom out', /* FIELD_LABEL */ + [STRING.FIELD_LABEL_ACCURACY_BY_SPECIES]: 'Accuracy by species', + [STRING.FIELD_LABEL_ACCURACY]: 'Accuracy', [STRING.FIELD_LABEL_ADDED_AT]: 'Added at', [STRING.FIELD_LABEL_ALGORITHM_URI]: 'Algorithm source', + [STRING.FIELD_LABEL_ALGORITHM]: 'Algorithm', [STRING.FIELD_LABEL_AVG_TEMP]: 'Avg temp', + [STRING.FIELD_LABEL_BEST_MODEL]: 'Best model', [STRING.FIELD_LABEL_BEST_SCORE]: 'Best score', [STRING.FIELD_LABEL_CAPTURE_SET]: 'Capture set', [STRING.FIELD_LABEL_OCCURRENCE_SET]: 'Occurrence set', @@ -465,6 +481,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.FIELD_LABEL_COMMENT]: 'Comment', [STRING.FIELD_LABEL_CONNECTION_STATUS]: 'Connection status', [STRING.FIELD_LABEL_CREATED_AT]: 'Created at', + [STRING.FIELD_LABEL_CROPS_WITHOUT_EMBEDDING]: 'Crops waiting for embeddings', [STRING.FIELD_LABEL_DATA_SOURCE_CAPTURES]: 'Data source captures', [STRING.FIELD_LABEL_DATA_SOURCE]: 'Data source', [STRING.FIELD_LABEL_DATE_OBSERVED]: 'Date observed', @@ -479,6 +496,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.FIELD_LABEL_EMAIL]: 'Email', [STRING.FIELD_LABEL_ENDPOINT]: 'Endpoint URL', [STRING.FIELD_LABEL_ERRORS]: 'Errors', + [STRING.FIELD_LABEL_EVALUATION_SET]: 'Evaluation set', [STRING.FIELD_LABEL_EXAMPLE]: 'Example', [STRING.FIELD_LABEL_FILE_SIZE]: 'File size', [STRING.FIELD_LABEL_FILENAME]: 'Filename', @@ -511,6 +529,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.FIELD_LABEL_NEW_PASSWORD]: 'New password', [STRING.FIELD_LABEL_NUM_PIPELINES_REGISTERED]: 'Pipelines registered', [STRING.FIELD_LABEL_OCCURRENCES]: 'Occurrences', + [STRING.FIELD_LABEL_OCCURRENCES_SCORED]: 'Occurrences scored', [STRING.FIELD_LABEL_PASSWORD_CURRENT]: 'Current password', [STRING.FIELD_LABEL_PASSWORD_NEW]: 'New password', [STRING.FIELD_LABEL_PASSWORD]: 'Password', @@ -547,6 +566,8 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.FIELD_LABEL_TOTAL_SIZE]: 'Total size', [STRING.FIELD_LABEL_TRAINING_IMAGES]: 'Reference images', [STRING.FIELD_LABEL_TYPE]: 'Type', + [STRING.FIELD_LABEL_SPECIES]: 'Species', + [STRING.FIELD_LABEL_TRAINING_IMAGES_READY]: 'Training images ready', [STRING.FIELD_LABEL_UPDATED_AT]: 'Updated at', [STRING.FIELD_LABEL_USER]: 'User', [STRING.FIELD_LABEL_VERSION_NAME]: 'Version name', @@ -715,6 +736,8 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { 'A monitoring station is a location where a device is deployed to collect captures of insects in the wild.', [STRING.TOOLTIP_DEVICE]: 'A device type is the type of equipment or camera used for collecting captures. One or many deployments can be connected to a device type. Device type refers to the model version, category or description of a kind of hardware, not the serial number of an individual device.', + [STRING.TOOLTIP_EVALUATION_SET]: + 'An evaluation set is a fixed list of verified occurrences. Two models can only be compared if they were scored on the same set.', [STRING.TOOLTIP_JOB]: 'A job is a task that requires time to complete and runs in the background. Examples include processing captures, syncing captures, and generating exports.', [STRING.TOOLTIP_LATEST_JOB_STATUS]: @@ -753,6 +776,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.AGREEMENT_EXACT]: 'Agreement (exact taxon)', [STRING.AGREEMENT_KAPPA]: "Cohen's kappa (beyond chance)", [STRING.ALGORITHMS]: 'Algorithms', + [STRING.ALGORITHMS_AND_PERFORMANCE]: 'Algorithms & performance', [STRING.ANONYMOUS_USER]: 'Anonymous user', [STRING.APPLY_ID_SHORT]: 'Apply', [STRING.APPLY_ID]: 'Apply ID', @@ -764,6 +788,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.CONNECTING]: 'Connecting...', [STRING.DEFAULT]: 'Default', [STRING.DETAILS]: 'Details', + [STRING.EVALUATION]: 'Evaluation', [STRING.EXTERNAL_RESOURCES]: 'External resources', [STRING.FORGOT_PASSWORD_DETAILS]: `No worries, we'll send you reset instructions.`, [STRING.FORGOT_PASSWORD]: 'Forgot password?',