diff --git a/docs/format/report.md b/docs/format/report.md index e4b8ae87..b87a45d1 100644 --- a/docs/format/report.md +++ b/docs/format/report.md @@ -188,4 +188,4 @@ stored in [AWS Timestream], please see [Storage Schema]. [AWS Timestream]: https://aws.amazon.com/timestream -[Storage Schema]: https://github.com/Brightspace/test-reporting-action/blob/main/docs/storage-schema.md +[Storage Schema]: ./storage-schema.md diff --git a/docs/format/storage-schema.md b/docs/format/storage-schema.md new file mode 100644 index 00000000..30f9c394 --- /dev/null +++ b/docs/format/storage-schema.md @@ -0,0 +1,173 @@ +# Storage Schema + +All report data once submitted to the back-end is stored in [AWS Timestream]. +The data is split across 2 tables ([`summary`](#summary) and +[`details`](#details)) to better store common items, reduce duplication and +allow for more targeted queries. Below is a breakdown of the various tables and +their columns as well as the corresponding JSON path within the report format +the data is sourced from. The current `measure_name` for both the +[`summary`](#summary) and [`details`](#details) tables is `report_v3_bc` — the +`_bc` suffix indicates the `details` records still carry deprecated dimensions +(`experience`, `type`, `tool`, `timeout`) for backwards compatibility. Once +those dimensions are removed the suffix will be dropped to `report_v3` on both +tables. Old format records will be marked with `report_v2`, `report_v2_bc` or +`*_test_run` and can require special handling. See notes under various +**Removed** sections for details. + +> [!NOTE] + The storage schema is always based on the [latest report schema] and uses the + [auto upgrading report construct] to make sure it's always the latest schema. + +## Tables + +### `summary` + +This table contains all of the information that is common to a test run as well +as some rolled up counts of various test statuses. + +#### Timestamp + +* `time` (`TIMESTAMP`): Stored with **millisecond** precision, sourced from + report JSON `report.summary.started`. + +#### Measures + +* `duration_total` (`BIGINT`): Stored as **milliseconds** from report JSON + `report.summary.duration.total`. +* `status` (`VARCHAR`): Will either be `passed` or `failed`, sourced from report + JSON `report.summary.status`. +* `count_passed` (`BIGINT`): Sourced from report JSON + `report.summary.count.passed`. +* `count_failed` (`BIGINT`): Sourced from report JSON + `report.summary.count.failed`. +* `count_skipped` (`BIGINT`): Sourced from report JSON + `report.summary.count.skipped`. +* `count_flaky` (`BIGINT`): Sourced from report JSON + `report.summary.count.flaky`. + +> [!WARNING] + Anything marked with **[deprecated]** should be moved away from. Sending of + this column will be removed in the near future. + +##### Removed + +* `total_duration` (`BIGINT`): Starting with records that have `measure_name` of + `report_v2` this field will no longer be present. Please use `duration_total` + instead. If wanting to get data from records prior to `report_v2` please use + `COALESCE(total_duration, duration_total)` to get a reasonable value. + +#### Dimensions + +> [!NOTE] + All dimensions are stored as `VARCHAR` as it is the only format available. + +* `report_id`: Sourced from report JSON `report.id`. +* `github_organization`: Sourced from report JSON + `report.summary.github.organization`. +* `github_repository`: Sourced from report JSON + `report.summary.github.repository`. +* `github_workflow`: Sourced from report JSON `report.summary.github.workflow`. +* `github_run_id`: Sourced from report JSON `report.summary.github.runId`. +* `github_run_attempt`: Sourced from report JSON + `report.summary.github.runAttempt`. +* `git_branch`: Sourced from report JSON `report.summary.git.branch`. +* `git_sha`: Sourced from report JSON `report.summary.git.sha`. +* `operating_system`: Will be one of `windows`, `linux` or `mac`, sourced from + report JSON `report.summary.operatingSystem`. +* `framework`: Sourced from report JSON `report.summary.framework`. +* `lms_build_number` (`NULLABLE`): Sourced from report JSON + `report.summary.lms.buildNumber`. +* `lms_instance_url` (`NULLABLE`): Sourced from report JSON + `report.summary.lms.instanceUrl`. + +> [!WARNING] + Anything marked with **[deprecated]** should be moved away from. Sending of + this column will be removed in the near future. + +### `details` + +This table contains information about each individual test that was run. You can +map to the [`summary`](#summary) data via the `report_id` if you need a +combination of the data. + +#### Timestamp + +* `time` (`TIMESTAMP`): Stored with **millisecond** precision, sourced from + report JSON `report.details[].started`. + +#### Measures + +* `duration_final` (`BIGINT`): Stored as **milliseconds**, sourced from report + JSON `report.details[].duration.final`. +* `duration_total` (`BIGINT`): Stored as **milliseconds**, sourced from report + JSON `report.details[].duration.total`. +* `retries` (`BIGINT`): Sourced from report JSON `report.details[].retries`. +* `status` (`VARCHAR`): Will be one of `passed`, `skipped` or `failed`, sourced + from report JSON `report.details[].status`. +* `configuration_timeout` (`BIGINT`, `NULLABLE`): Stored as **milliseconds**, sourced + from report JSON `report.details[].configuration.timeout`. + +> [!WARNING] + Anything marked with **[deprecated]** should be moved away from. Sending of + this column will be removed in the near future. + +##### Removed + +* `duration` (`BIGINT`): Starting with records that have `measure_name` of + `report_v2` this field will no longer be present. Please use `duration_final` + instead. If wanting to get data from records prior to `report_v2` please use + `COALESCE(duration, duration_final)` to get a reasonable value. +* `total_duration` (`BIGINT`): Starting with records that have `measure_name` of + `report_v2` this field will no longer be present. Please use `duration_total` + instead. If wanting to get data from records prior to `report_v2` please use + `COALESCE(total_duration, duration_total)` to get a reasonable value. + +#### Dimensions + +> [!NOTE] + All dimensions are stored as `VARCHAR` as it is the only format available. + +* `report_id`: Sourced from report JSON `report.id`. +* `name`: Sourced from report JSON `report.details[].name`. +* `location_file`: Sourced from report JSON `report.details[].location.file`. +* `location_line` (`NULLABLE`): Sourced from report JSON + `report.details[].location.line`. +* `location_column` (`NULLABLE`): Sourced from report JSON + `report.details[].location.column`. +* `browser` (`NULLABLE`): Can be one of `chrome`, `chromium`, `firefox`, + `webkit`, `safari` or `edge`, sourced from report JSON + `report.details[].browser`. +* `github_codeowners` (`NULLABLE`): A comma-separated list sourced from report + JSON `report.details[].github.codeowners`. +* `type` (`NULLABLE`) **[deprecated]**: Replaced by `taxonomy_type`. Sourced + from report JSON `report.details[].taxonomy.type`. +* `taxonomy_type` (`NULLABLE`): Sourced from report JSON + `report.details[].taxonomy.type`. +* `experience` (`NULLABLE`) **[deprecated]**: No longer part of the report + schema starting with v3. For v1/v2 reports this is sourced from the original + JSON at `report.details[].experience` before upgrade and injected onto the + outgoing record. Records without this dimension will appear once the source + reports are exclusively v3 or newer. +* `tool` (`NULLABLE`) **[deprecated]**: Replaced by `taxonomy_tool`. Sourced + from report JSON `report.details[].taxonomy.tool`. +* `taxonomy_tool` (`NULLABLE`): Sourced from report JSON + `report.details[].taxonomy.tool`. +* `timeout` (`NULLABLE`) **[deprecated]**: Replaced by the `configuration_timeout` + measure. Stored as **milliseconds**, sourced from report JSON + `report.details[].configuration.timeout`. + +> [!WARNING] + Anything marked with **[deprecated]** should be moved away from. Sending of + this column will be removed in the near future. + +##### Removed + +* `location`: Starting with records that have `measure_name` of `report_v2` this + field will no longer be present. Please use `location_file` instead. If + wanting to get data from records prior to `report_v2` please use + `COALESCE(location, location_file)` to get a reasonable value. + + +[AWS Timestream]: https://aws.amazon.com/timestream +[latest report schema]: https://github.com/Brightspace/test-reporting-node/tree/main/schemas/report +[auto upgrading report construct]: https://github.com/Brightspace/test-reporting-node/blob/main/src/helpers/report.cjs diff --git a/docs/report-submitter.md b/docs/report-submitter.md new file mode 100644 index 00000000..ff544f65 --- /dev/null +++ b/docs/report-submitter.md @@ -0,0 +1,28 @@ +# Report Submitter + +## When to Use + +In most cases `ReportSubmitter` should not be used. To upload reports from Github use the [GitHub Action]. +It is used when the report needs to be uploaded outside of Github Actions. + +## Quick Start + +```js +import { ReportBuilder } from 'd2l-test-reporting/helpers/report-builder.js'; +import { ReportSubmitter } from 'd2l-test-reporting/helpers/report-submitter.js'; + +const logger = { + info: (msg) => console.log(msg), + warning: (msg) => console.warn(msg), + error: (msg) => console.error(msg), + location: (msg, loc) => console.log(`${msg}: ${loc}`) +}; + +const report = new ReportBuilder('your-framework-name', logger); +const submitter = new ReportSubmitter(logger); // AWS credentials are read from environment variables +await submitter.submit(report); +``` + + + +[GitHub Action]: https://github.com/Brightspace/test-reporting-action diff --git a/package-lock.json b/package-lock.json index e490bf2a..62762350 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "6.5.0", "license": "Apache-2.0", "dependencies": { + "@aws-sdk/client-timestream-write": "^3.1127.0", "@wdio/reporter": "^9", "@web/test-runner-core": "^1", "ajv": "^8", @@ -138,6 +139,290 @@ "dev": true, "license": "MIT" }, + "node_modules/@aws-sdk/client-timestream-write": { + "version": "3.1127.0", + "integrity": "sha512-nNsSDM6jjwq+xPyv4OTa3aTrpfAsYz8H4xmtpuN7Cv5xnAxFLe9xBvboRPsCLrIdRUcR/j7M96vDaHfBm8/Cyg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-node": "^3.972.82", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.30", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.977.9", + "integrity": "sha512-reqPFEQrZxDZpeGj4PFMepBeR5LGYHRqq/L0motTzgFkCRBA4rFdaVXDSLYyGHhxVz7sT2PDnPN9CluGSfgyJA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.5", + "@aws-sdk/xml-builder": "^3.972.40", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.33.3", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.17.2", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.70", + "integrity": "sha512-H404B7dJl2mCrBqahDEYsanB0xhdDp6tXnXcTUnXmmpy2Q3J0Ho0bUajZ2jr/RdwzCyS59Gi8xXIFwPLGBl6Uw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.72", + "integrity": "sha512-X98zYOrVOeuosCX+6ktf29FC2N2GHPLia7qv6mzPzTc+RPAuHWCDS++Z6JK7eGYqb/v6uaW7bAXaOvDBfol+0w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.15", + "integrity": "sha512-Rykg6s5ceBuynMOGWgoowO4N+27JfnqXAnVaSunZl0hOO1XodSrxGNz6sCEbnmS0lAfQZDKyb3fbr46gSuv6Sg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-env": "^3.972.70", + "@aws-sdk/credential-provider-http": "^3.972.72", + "@aws-sdk/credential-provider-login": "^3.972.77", + "@aws-sdk/credential-provider-process": "^3.972.70", + "@aws-sdk/credential-provider-sso": "^3.973.14", + "@aws-sdk/credential-provider-web-identity": "^3.972.76", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.77", + "integrity": "sha512-Jb59xfEISoN5mmbnA+HYqdtrSX3CgCtJoof+V5D8/TgUI56W63GEEd5Y58WijU3Ou6+WEgaLD1feVzaRXV5IDQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.82", + "integrity": "sha512-znDkEOGXB8W3kG1LJUKP3foBZY/9qLM0eil/DxWXSp37XsdsRLQHE/d/OaCGGVgKpA6znR38h/+INk8do1FjiA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.70", + "@aws-sdk/credential-provider-http": "^3.972.72", + "@aws-sdk/credential-provider-ini": "^3.973.15", + "@aws-sdk/credential-provider-process": "^3.972.70", + "@aws-sdk/credential-provider-sso": "^3.973.14", + "@aws-sdk/credential-provider-web-identity": "^3.972.76", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.70", + "integrity": "sha512-2ry03fGRJr4sV3jI+ocjj5JqALnFD6ymM5KiNCDZMvq8bX2GSbE0vji4aM43TVCl2nXqqLRZaUxdq/KeWRAY4Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.14", + "integrity": "sha512-jkhg/8ocAAoc0RFyLMhCw+/zZh7gystQgd4F4hznNa8P4Cc501PQmxd+jGLiMHodPJ+7Zv/3znM62gZojyasmA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/token-providers": "3.1116.0", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.76", + "integrity": "sha512-d3AGyVu759PGr35mEB2s22xxlNEA5rpdxtSPJthfPFJvoQ8dt357iVPECqWfUxXp1toJAvKmbtcIYVGigaGsCA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/endpoint-cache": { + "version": "3.972.11", + "integrity": "sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw==", + "license": "Apache-2.0", + "dependencies": { + "mnemonist": "0.38.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery": { + "version": "3.972.30", + "integrity": "sha512-0hmD7/NG2NoVOVHvUb6rtY5POQYr+/9gdHvrYhIBA1zmCqKOBd5Gz3dL+iZ15FHOJbs/5kLplGoePc8PHTlzYA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/endpoint-cache": "^3.972.11", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.44", + "integrity": "sha512-NhEgryjlBF9w38ZXqGymQV28IhkYa1mKhlbYnqIis57AYwWGVYfUPgg/qC2rLRqOUfblxx++irvju10kVTa8Vw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/signature-v4-multi-region": "^3.996.46", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.46", + "integrity": "sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.5", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1116.0", + "integrity": "sha512-ygIivKqh8aHzNkucOCXHyIBgBpLPfrSI0mCqXF+vLBsPTUKqj0VSqAY0GFPe7lQl4HntjOcQ+KSyS7oUV2C54Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.5", + "integrity": "sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.40", + "integrity": "sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@azure/abort-controller": { "version": "2.2.0", "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", @@ -2834,9 +3119,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3234,9 +3516,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3257,9 +3536,6 @@ "arm" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3280,9 +3556,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3303,9 +3576,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3326,9 +3596,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3349,9 +3616,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3809,9 +4073,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3825,9 +4086,6 @@ "arm" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3841,9 +4099,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3857,9 +4112,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3873,9 +4125,6 @@ "loong64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3889,9 +4138,6 @@ "loong64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3905,9 +4151,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3921,9 +4164,6 @@ "ppc64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3937,9 +4177,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3953,9 +4190,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3969,9 +4203,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3985,9 +4216,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4001,9 +4229,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4155,6 +4380,81 @@ "node": ">=4" } }, + "node_modules/@smithy/core": { + "version": "3.33.3", + "integrity": "sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.5.2", + "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.8.0", + "integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.12.1", + "integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.7.3", + "integrity": "sha512-7ImGm+FkHRLcBaRttIAMZ6bzJZWb2cJGoYjq46F2UjycujWzrL9GEN9h4w7eQyXJYnltrUhxbbieBAIRrdqpow==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.18.0", + "integrity": "sha512-CgB6HHWer/vrKps24ulRIbpcpb7K4xAU7SkZ7YHzBPlwHsvsrCJFEXK421s+cJzX+ZrqtA/TuU5w1HzI7k9N8A==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@stylistic/eslint-plugin": { "version": "5.10.0", "integrity": "sha512-nPK52ZHvot8Ju/0A4ucSX1dcPV2/1clx0kLcH5wDmrE4naKso7TUC/voUyU1O9OTKTrR6MYip6LP0ogEMQ9jPQ==", @@ -4818,9 +5118,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4834,9 +5131,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4850,9 +5144,6 @@ "loong64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4866,9 +5157,6 @@ "loong64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4882,9 +5170,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4898,9 +5183,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4914,9 +5196,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4930,9 +5209,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4946,9 +5222,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4962,9 +5235,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6966,6 +7236,11 @@ "dev": true, "license": "MIT" }, + "node_modules/bowser": { + "version": "2.14.1", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "5.0.9", "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", @@ -7587,19 +7862,6 @@ "node": ">=12.13.0" } }, - "node_modules/chromium-bidi": { - "version": "14.0.0", - "integrity": "sha512-9gYlLtS6tStdRWzrtXaTMnqcM4dudNegMXJxkR0I/CXObHalYeYcAMPrL19eroNZHtJ8DQmu1E+ZNOYu/IXMXw==", - "extraneous": true, - "license": "Apache-2.0", - "dependencies": { - "mitt": "^3.0.1", - "zod": "^3.24.1" - }, - "peerDependencies": { - "devtools-protocol": "*" - } - }, "node_modules/ci-info": { "version": "4.4.0", "integrity": "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==", @@ -8611,12 +8873,6 @@ "node": ">=8" } }, - "node_modules/devtools-protocol": { - "version": "0.0.1608973", - "integrity": "sha512-Tpm17fxYzt+J7VrGdc1k8YdRqS3YV7se/M6KeemEqvUbq/n7At1rWVuXMxQgpWkdwSdIEKYbU//Bve+Shm4YNQ==", - "extraneous": true, - "license": "BSD-3-Clause" - }, "node_modules/dezalgo": { "version": "1.0.4", "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", @@ -14618,6 +14874,14 @@ "node": ">=10" } }, + "node_modules/mnemonist": { + "version": "0.38.3", + "integrity": "sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==", + "license": "MIT", + "dependencies": { + "obliterator": "^1.6.1" + } + }, "node_modules/mocha": { "version": "12.0.0", "integrity": "sha512-NYNh5IFt6WYqm9bi4601m7vix8MZdXC0DwS4gY6WhXO2RgJWhivhISVmq1oklCif18OSI9l+vx5Mdm5oh1XGiQ==", @@ -15274,6 +15538,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/obliterator": { + "version": "1.6.1", + "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==", + "license": "MIT" + }, "node_modules/obug": { "version": "2.1.4", "integrity": "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==", @@ -16162,45 +16431,6 @@ "node": ">=6" } }, - "node_modules/puppeteer-core": { - "version": "24.43.1", - "integrity": "sha512-T5ScUMAsmhdNbgDR41AGESYeS6V9MSgetkSnVhhW+gXvzC42VesKCn5ld87gAZDJ6vLHL9GkRvY9WtQWSnwFbw==", - "extraneous": true, - "license": "Apache-2.0", - "dependencies": { - "@puppeteer/browsers": "2.13.2", - "chromium-bidi": "14.0.0", - "debug": "^4.4.3", - "devtools-protocol": "0.0.1608973", - "typed-query-selector": "^2.12.2", - "webdriver-bidi-protocol": "0.4.1", - "ws": "^8.20.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/puppeteer-core/node_modules/ws": { - "version": "8.21.3", - "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", - "extraneous": true, - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/pure-rand": { "version": "7.0.1", "integrity": "sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==", @@ -18357,7 +18587,6 @@ "node_modules/tslib": { "version": "2.8.1", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, "license": "0BSD" }, "node_modules/tsscmp": { @@ -18916,12 +19145,6 @@ "node": ">=18.20.0" } }, - "node_modules/webdriver-bidi-protocol": { - "version": "0.4.1", - "integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==", - "extraneous": true, - "license": "Apache-2.0" - }, "node_modules/webdriver/node_modules/@types/ws": { "version": "8.18.1", "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", diff --git a/package.json b/package.json index e1e44b79..7bb5a4b4 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "./helpers/report.js": "./src/helpers/report.cjs", "./helpers/report-configuration.js": "./src/helpers/report-configuration.cjs", "./helpers/report-builder.js": "./src/helpers/report-builder.cjs", + "./helpers/report-submitter.js": "./src/helpers/report-submitter.cjs", "./reporters/jest.js": "./src/reporters/jest.cjs", "./reporters/mocha.js": "./src/reporters/mocha.cjs", "./reporters/node.js": "./src/reporters/node.js", @@ -49,6 +50,7 @@ "test:unit": "node --test --test-reporter=spec --test-reporter-destination=stdout --test-reporter=./src/reporters/node.js --test-reporter-destination=./d2l-test-report.json \"test/unit/*.test.js\"" }, "dependencies": { + "@aws-sdk/client-timestream-write": "^3.1127.0", "@wdio/reporter": "^9", "@web/test-runner-core": "^1", "ajv": "^8", diff --git a/src/helpers/report-submitter.cjs b/src/helpers/report-submitter.cjs new file mode 100644 index 00000000..d2d69196 --- /dev/null +++ b/src/helpers/report-submitter.cjs @@ -0,0 +1,322 @@ +import { MeasureValueType, TimestreamWriteClient, TimeUnit, WriteRecordsCommand } from '@aws-sdk/client-timestream-write'; + +const { BIGINT, VARCHAR, MULTI } = MeasureValueType; +const { MILLISECONDS } = TimeUnit; +// const repoSettingsDocUrl = 'https://github.com/Brightspace/repo-settings/blob/main/docs/test-reporting.md#analytics'; // TODO Still needed? + +// TODO +// - Issue this is a public repo... Anything a concern? +// - Figure out how to use the Report class to provide the data +// - Review finalizer method https://github.com/Brightspace/test-reporting-action/blob/main/src/report.js#L331 +// Concerns: duplicates code from https://github.com/Brightspace/test-reporting-action/blob/main/src/report.js#L8 +// - Could this class be used for the gha? + +/** + * AWS credentials must be set as environment variable. These credentials must be for a role that can write to Timestream + */ +class ReportSubmitter { + #logger; + #experienceMaps; + + constructor(logger) { + this.debug = false; + this.dryRun = false; + this.#logger = logger; + this.region = 'us-east-1'; + this.databaseName = 'test_reporting'; + this.#experienceMaps = new WeakMap(); + + /* + const logger = { + info: (msg) => console.log(msg), + warning: (msg) => console.warn(msg), + error: (msg) => console.error(msg), + location: (msg, loc) => console.log(`${msg}: ${loc}`) + }; + */ + } + + async submit(report) { + this.#logger.info('Submit report'); + this.#logger.info('Generate summary write request'); + + let reportJson = report.toJSON(); + const summaryWriteRequest = this.#makeSummaryWriteRequest(reportJson); + + this.#logger.info('Generate detail write requests'); + + const detailWriteRequests = this.#makeDetailWriteRequests(reportJson); + + reportJson = null; + + if (this.#experienceMaps.has(report)) { + this.#logger.info('Restoring experience dimension onto detail records'); + } + + this.#applyExperienceDimensions(report, detailWriteRequests); + + this.#logger.info('Merge write requests'); + + const writeRequests = [ + summaryWriteRequest, + ...detailWriteRequests + ]; + + if (this.dryRun) { + this.#logger.info('Dry run, skipping records submit'); + + return; + } + + this.#logger.info('Executing write requests'); + + await this.#writeTimestream(writeRequests); + } + + #applyExperienceDimensions(report, detailWriteRequests) { + const map = this.#experienceMaps.get(report); + + if (!map) { + return; + } + + this.#experienceMaps.delete(report); + + for (const request of detailWriteRequests) { + for (const record of request.Records) { + const name = record.Dimensions.find(d => d.Name === 'name')?.Value; + const locationFile = record.Dimensions.find(d => d.Name === 'location_file')?.Value; + const experience = map.get(`${name}|${locationFile}`); + + if (experience) { + record.Dimensions.push({ Name: 'experience', Value: experience }); + } + } + } + }; + + #makeSummaryWriteRequest(report) { + const { id, version, summary } = report; + const { + github: { + organization, + repository, + workflow, + runId, + runAttempt + }, + git: { + branch, + sha + }, + operatingSystem, + framework, + started, + duration: { + total + }, + status, + count: { + passed, + failed, + skipped, + flaky + }, + lms + } = summary; + + const dimensions = [ + { Name: 'report_id', Value: id }, + { Name: 'github_organization', Value: organization }, + { Name: 'github_repository', Value: repository }, + { Name: 'github_workflow', Value: workflow }, + { Name: 'github_run_id', Value: runId.toString() }, + { Name: 'github_run_attempt', Value: runAttempt.toString() }, + { Name: 'git_branch', Value: branch }, + { Name: 'git_sha', Value: sha }, + { Name: 'operating_system', Value: operatingSystem }, + { Name: 'framework', Value: framework } + ]; + + if (lms) { + const { buildNumber, instanceUrl } = lms; + + if (buildNumber) { + dimensions.push({ Name: 'lms_build_number', Value: buildNumber }); + } + + if (instanceUrl) { + dimensions.push({ Name: 'lms_instance_url', Value: instanceUrl }); + } + } + + return { + DatabaseName: this.databaseName, + TableName: 'summary', + Records: [{ + Version: 1, + Time: (Date.parse(started)).toString(), + TimeUnit: MILLISECONDS, + // `_bc` suffix matches the `details` table and signals that the + // records are emitted alongside deprecated detail dimensions. Drop the + // suffix once the deprecated dimensions are removed. + MeasureName: `report_v${version}_bc`, + MeasureValueType: MULTI, + MeasureValues: [ + { Name: 'duration_total', Value: total.toString(), Type: BIGINT }, + { Name: 'status', Value: status, Type: VARCHAR }, + { Name: 'count_passed', Value: passed.toString(), Type: BIGINT }, + { Name: 'count_failed', Value: failed.toString(), Type: BIGINT }, + { Name: 'count_skipped', Value: skipped.toString(), Type: BIGINT }, + { Name: 'count_flaky', Value: flaky.toString(), Type: BIGINT } + ], + Dimensions: dimensions + }] + }; + }; + + #makeDetailRecord(detail) { + const { + name, + started, + location, + retries, + config, + configuration, + github, + duration: { + total, + final + }, + status, + browser, + taxonomy + } = detail; + const { file, line, column } = location ?? {}; + const { timeout } = configuration ?? config ?? {}; + const { codeowners } = github ?? {}; + const { type, tool } = taxonomy ?? {}; + const measures = [ + { Name: 'duration_final', Value: final.toString(), Type: BIGINT }, + { Name: 'duration_total', Value: total.toString(), Type: BIGINT }, + { Name: 'retries', Value: retries.toString(), Type: BIGINT }, + { Name: 'status', Value: status, Type: VARCHAR } + ]; + const dimensions = [ + { Name: 'name', Value: name } + ]; + + if (file) { + dimensions.push({ Name: 'location_file', Value: file }); + } + + if (timeout) { + // kept for backwards compat. Once all dashboards are updated will be removed + dimensions.push({ Name: 'timeout', Value: timeout.toString() }); + ///////////////////////////////////////////////////////////////////////////// + measures.push({ Name: 'configuration_timeout', Value: timeout.toString(), Type: BIGINT }); + } + + if (line) { + dimensions.push({ Name: 'location_line', Value: line.toString() }); + } + + if (column) { + dimensions.push({ Name: 'location_column', Value: column.toString() }); + } + + if (browser) { + dimensions.push({ Name: 'browser', Value: browser }); + } + + if (type) { + // kept for backwards compat. Once all dashboards are updated will be removed + dimensions.push({ Name: 'type', Value: type }); + ///////////////////////////////////////////////////////////////////////////// + dimensions.push({ Name: 'taxonomy_type', Value: type }); + } + + if (tool) { + // kept for backwards compat. Once all dashboards are updated will be removed + dimensions.push({ Name: 'tool', Value: tool }); + ///////////////////////////////////////////////////////////////////////////// + dimensions.push({ Name: 'taxonomy_tool', Value: tool }); + } + + if (codeowners) { + dimensions.push({ Name: 'github_codeowners', Value: codeowners.join(',') }); + } + + return { + Time: (Date.parse(started)).toString(), + TimeUnit: MILLISECONDS, + MeasureValues: measures, + Dimensions: dimensions + }; + }; + + #makeDetailWriteRequests(report) { + const { id, version, details } = report; + const batchSize = 100; + const writeRequests = Array.from( + { length: Math.ceil(details.length / batchSize) }, + (v, i) => { + const detailRecordBatch = details + .slice(i * batchSize, i * batchSize + batchSize) + .map(this.#makeDetailRecord); + + return { + DatabaseName: this.databaseName, + TableName: 'details', + Records: detailRecordBatch, + CommonAttributes: { + Version: 1, + // `_bc` suffix signals that records still carry deprecated + // dimensions (`experience`, `type`, `tool`) for backwards + // compatibility. Drop the suffix once those are removed. + MeasureName: `report_v${version}_bc`, + MeasureValueType: MULTI, + Dimensions: [ + { Name: 'report_id', Value: id, Type: VARCHAR } + ] + } + }; + } + ); + + return writeRequests; + }; + + async #writeTimestream(credentials, requests) { + const client = new TimestreamWriteClient({ credentials, region: this.region }); + + for (const [index, request] of requests.entires()) { + try { + this.#logger.info(`Sending batch ${index + 1} of ${requests.length} (${request.Records.length} records)`); + + if (this.debug) { + this.#logger.info(`${JSON.stringify(request, null, 2)}\n`); + } + const command = new WriteRecordsCommand(request); + await client.send(command); + } catch (err) { + const { name, message, $metadata = {}, RejectedRecords } = err; + const { httpStatusCode = '[unknown]', requestId = '[unknown]' } = $metadata; + + this.#logger.error(`AWS ${name} on batch ${index + 1}/${requests.length} (HTTP ${httpStatusCode}, request ${requestId}): ${message}`); + + if (Array.isArray(RejectedRecords)) { + for (const rejected of RejectedRecords) { + const record = JSON.stringify(request.Records[rejected.RecordIndex]); + + this.#logger.error(`Rejected record ${rejected.RecordIndex} (${rejected.Reason}): ${record}`); + } + } + + throw new Error('Unable to submit write requests'); + } + } + } +} + +module.exports = { ReportSubmitter }; diff --git a/src/helpers/report-submitter.d.cts b/src/helpers/report-submitter.d.cts new file mode 100644 index 00000000..182af4ef --- /dev/null +++ b/src/helpers/report-submitter.d.cts @@ -0,0 +1,16 @@ +import { Report } from './report.cjs'; + +export interface Logger { + info(message: string): void; + warning(message: string): void; + error(message: string): void; + location(message: string, location: string): void; +} + +export interface ReportSubmitter { + logger: Logger; +} + +export interface ReportSubmitterSubmit { + report: Report; +} diff --git a/test/unit/report-submitter.test.js b/test/unit/report-submitter.test.js new file mode 100644 index 00000000..e69de29b