From dd774e54b05e662ac2bbead012353592274b723a Mon Sep 17 00:00:00 2001 From: muhammadali-imran Date: Sat, 26 Sep 2026 03:07:17 +0500 Subject: [PATCH 1/5] google drive connection interface --- backend/.env.example | 7 +- backend/README.md | 8 +- backend/docs/API.md | 5 +- backend/docs/ARCHITECTURE.md | 6 +- backend/package-lock.json | 230 +++++++++++++++++- backend/package.json | 1 + backend/src/config/env.js | 13 +- .../src/controllers/submissionController.js | 3 - backend/src/docs/openapi.js | 1 - backend/src/models/Submission.js | 1 + backend/src/models/storedFile.js | 1 + backend/src/repositories/submissionRepo.js | 1 + backend/src/routes/submissionRoutes.js | 6 +- .../src/services/storage/googleDriveDriver.js | 83 +++++++ backend/src/services/storage/index.js | 16 +- backend/src/services/storage/localDriver.js | 4 +- backend/src/services/submissionService.js | 36 +-- backend/tests/api.test.js | 21 +- 18 files changed, 386 insertions(+), 57 deletions(-) create mode 100644 backend/src/services/storage/googleDriveDriver.js diff --git a/backend/.env.example b/backend/.env.example index 61caf01..b64233f 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -27,9 +27,14 @@ ADMIN_PASSWORD= ADMIN_FIRST_NAME=Admin ADMIN_SURNAME=Signpak -# Where uploaded files go. Only "local" exists today; a Google Drive driver plugs in here later. +# Where uploaded files go. For Google Drive, share the target folder with the service account email. STORAGE_DRIVER=local +# Submission recordings are append-only archives. Set this to gdrive for Drive storage. +ARCHIVE_STORAGE_DRIVER=local UPLOAD_DIR=uploads +# Required when STORAGE_DRIVER=gdrive. Store the whole service-account JSON as one environment value. +# GOOGLE_SERVICE_ACCOUNT_JSON={"type":"service_account",...} +# GOOGLE_DRIVE_FOLDER_ID=your-shared-drive-folder-id MAX_VIDEO_UPLOAD_MB=300 MAX_RECORDING_UPLOAD_MB=100 SUBMISSION_COOLDOWN_MS=30000 diff --git a/backend/README.md b/backend/README.md index b03f66a..3982722 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,6 +1,6 @@ # SignPakCommons: backend -Express + Node.js + MongoDB (Mongoose). Cookie-based auth, local file storage (Google Drive ready), layered architecture. +Express + Node.js + MongoDB (Mongoose). Cookie-based auth, local base-video storage, optional append-only Google Drive recording archives, layered architecture. ## Run it @@ -67,7 +67,7 @@ Codes: `BAD_REQUEST` 400, `UNAUTHORIZED` 401, `FORBIDDEN` 403, `NOT_FOUND` 404, | `PATCH`/`DELETE /videos/:id` | admin | Moving to another category appends to the end of that path | | `GET /submissions` | logged in | Contributors: their own. Admin: all | | `POST /submissions` | logged in | `multipart/form-data`: `recording`, `videoId`, `trimStart`, `trimEnd`, `mirrored`, `duration` | -| `GET /submissions/:id/recording` | admin | Contributors can never play a submission back | +| `GET /submissions/:id/recording` | none | Deliberately unavailable; submissions are append-only archives | | `POST /contact` | anyone | Rate limited: 5 per hour per IP | | `GET /admin/users`, `/admin/stats?days=14`, `/admin/messages` | admin | | @@ -79,11 +79,11 @@ Codes: `BAD_REQUEST` 400, `UNAUTHORIZED` 401, `FORBIDDEN` 403, `NOT_FOUND` 404, **Admin.** Created or promoted from `ADMIN_EMAIL` / `ADMIN_PASSWORD` on every start. Signup ignores any `role` in the body. Changing `ADMIN_PASSWORD` later does not reset an existing admin's password. -**The cooldown rule.** This is a data-collection tool, not a one-shot quiz: a contributor can submit the same video more than once (more varied takes make better training data), but not back to back. `SUBMISSION_COOLDOWN_MS` (default 30s, in `.env`) is the minimum gap between two submissions for the same `(user, video)`. It is enforced atomically, not by a simple read-then-write: `repositories/submissionRepo.js#claimCooldown` does a conditional `findOneAndUpdate` with `upsert: true` against a `SubmissionCooldown` collection that has a unique index on `(user, video)`. If a live cooldown already exists, the upsert's insert path collides with that unique index (`E11000`), which is treated as "still cooling down" — so three simultaneous submissions for the same video still produce exactly one success (verified: `tests/api.test.js`'s concurrent-submit test). A submission that fails after claiming the cooldown (a storage or DB error) releases it, so a failed attempt never costs the contributor their next try. Regardless of timing, no submission can ever be edited or deleted once made, and only an admin can watch one back. +**The cooldown rule.** This is a data-collection tool, not a one-shot quiz: a contributor can submit the same video more than once (more varied takes make better training data), but not back to back. `SUBMISSION_COOLDOWN_MS` (default 30s, in `.env`) is the minimum gap between two submissions for the same `(user, video)`. It is enforced atomically, not by a simple read-then-write: `repositories/submissionRepo.js#claimCooldown` does a conditional `findOneAndUpdate` with `upsert: true` against a `SubmissionCooldown` collection that has a unique index on `(user, video)`. If a live cooldown already exists, the upsert's insert path collides with that unique index (`E11000`), which is treated as "still cooling down" — so three simultaneous submissions for the same video still produce exactly one success (verified: `tests/api.test.js`'s concurrent-submit test). A submission that fails after claiming the cooldown (a storage or DB error) releases it, so a failed attempt never costs the contributor their next try. Regardless of timing, no submission can ever be edited, deleted, or read back once archived. Admins receive archive metadata only. **Uploads.** Multer writes to `uploads/tmp`, the service checks the file's real type from its first bytes (MP4, WebM or MOV; JPEG, PNG or WebP for posters; the browser-supplied type is not trusted), then hands it to storage. Failures at any step leave no files behind. -**Storage and Google Drive.** MongoDB stores only `{ driver, key, mimeType, size }`. `services/storage/localDriver.js` documents a four-method contract (`save`, `stat`, `createReadStream`, `remove`). To add Google Drive: write `googleDriveDriver.js` with those four methods (Drive's `files.get` with `alt=media` accepts Range headers), register it in `services/storage/index.js`, and allow `gdrive` in `STORAGE_DRIVER`. Files already stored keep working, because each one remembers which driver holds it. +**Storage and Google Drive.** Base videos use the readable storage driver selected by `STORAGE_DRIVER`. Submission recordings use the separate append-only driver selected by `ARCHIVE_STORAGE_DRIVER`; set it to `gdrive` with `GOOGLE_SERVICE_ACCOUNT_JSON` and `GOOGLE_DRIVE_FOLDER_ID`. MongoDB stores the archive pointer and metadata, while the API never reads, updates, or deletes the archived recording. Existing file records keep working because each one remembers its driver. **Stats.** Per-category counts only include submissions whose video still exists. Totals and daily activity always count everything. Days are UTC. Per-day bucketing is done in code from a timestamps-only query; if submissions reach hundreds of thousands, move it to a MongoDB aggregation. diff --git a/backend/docs/API.md b/backend/docs/API.md index 2254b33..5dccae1 100644 --- a/backend/docs/API.md +++ b/backend/docs/API.md @@ -55,7 +55,7 @@ Errors use a structured object suitable for form-level and field-level messages: | GET | `/videos/:id/poster` | Public when published | Stream a published poster. | | POST/PATCH/DELETE | `/videos`, `/videos/:id` | Admin | Upload and manage base videos. | | GET/POST | `/submissions` | User | List or create contributor recordings. | -| GET | `/submissions/:id/recording` | Admin | Stream a contributor recording. | +| GET | `/submissions/:id/recording` | None | Deliberately unavailable; submission archives are append-only. | | POST | `/contact` | Public | Send a contact message. | | GET | `/admin/users` | Admin | List users. | | GET | `/admin/stats` | Admin | Read dashboard statistics. | @@ -72,5 +72,6 @@ The server validates file signatures, size limits, metadata, and route IDs. Clie - Anonymous users see only published videos assigned to a category. - Admins can list and manage draft, published, and unassigned videos. - Contributor submissions are private to their owner in `/submissions`. -- Only admins can stream submission recordings. +- Submission recordings are never readable through the API, including by admins. Admins can access the MongoDB submission record and its archive metadata. +- Archive paths follow `commons/{userId}_{sequence}/{category_slug}/{video_slug}/{video_slug}.{ext}`. A repeated submission for the same user and video increments `sequence` instead of overwriting the previous file. - Video reference files are public only when the corresponding video passes the same published visibility check. diff --git a/backend/docs/ARCHITECTURE.md b/backend/docs/ARCHITECTURE.md index f6c82fe..bda34ab 100644 --- a/backend/docs/ARCHITECTURE.md +++ b/backend/docs/ARCHITECTURE.md @@ -45,7 +45,9 @@ The authentication token is stored in the `httpOnly` `signpak_token` cookie. The MongoDB stores users, categories, videos, submissions, cooldown records, and contact messages. Video and recording bytes are stored through the storage driver; MongoDB stores a driver reference, key, MIME type, and size rather than file contents. -Published videos assigned to a category are visible to anonymous visitors. Their metadata, posters, and reference files can be read by the public demo and library. Draft or unassigned videos remain restricted to admins. Contributor recordings are never publicly playable; only admins can access a submission recording. +Published videos assigned to a category are visible to anonymous visitors. Their metadata, posters, and reference files can be read by the public demo and library. Draft or unassigned videos remain restricted to admins. Contributor recordings are never publicly playable. + +Submission recordings use a separate append-only archive path when `ARCHIVE_STORAGE_DRIVER=gdrive`: `commons/{userId}_{sequence}/{category_slug}/{video_slug}/{video_slug}.{ext}`. The API creates the archive object and stores its metadata in MongoDB, but exposes no read, update, or delete operation for the recording. Admin submission responses include the archive path for operational access; contributor responses do not. Uploads are first written to a temporary directory, checked by file signatures, and moved into storage only after validation. Failure cleanup removes temporary and already-stored files where necessary. @@ -56,7 +58,7 @@ Uploads are first written to a temporary directory, checked by file signatures, - SameSite HTTP-only JWT cookies plus origin verification for state-changing requests. - Global and operation-specific rate limits. - Zod validation for JSON, multipart fields, route IDs, and query parameters. -- Role checks for admin routes and submission recording playback. +- Role checks for admin routes and archive metadata access. - File type sniffing instead of trusting browser-provided MIME types. ## Testing and extension points diff --git a/backend/package-lock.json b/backend/package-lock.json index 2090c83..7050c92 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -20,6 +20,7 @@ "express-rate-limit": "^8.7.0", "express-validator": "^7.2.1", "fs-extra": "^11.4.1", + "googleapis": "^182.0.0", "helmet": "^8.3.0", "jsonwebtoken": "^9.0.3", "mongoose": "^7.8.12", @@ -128,7 +129,6 @@ "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "dev": true, "license": "MIT", "engines": { "node": ">= 14" @@ -362,6 +362,26 @@ "bare-path": "^3.0.0" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/basic-auth": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", @@ -406,6 +426,15 @@ "integrity": "sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==", "license": "MIT" }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -706,6 +735,15 @@ "url": "https://opencollective.com/express" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -977,6 +1015,12 @@ "node": ">= 8.0.0" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, "node_modules/fast-fifo": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", @@ -984,6 +1028,29 @@ "dev": true, "license": "MIT" }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -1083,6 +1150,18 @@ "node": ">= 6" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1139,6 +1218,34 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gaxios": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.3.1.tgz", + "integrity": "sha512-kB3rzJV7d9juLZh8/56QTXCwQfxyhdOMdyYk1HdQKFtF8TJTDTZQJtixWIwXdE9Jji91mC41DUNpjleo4L4eAQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-9.0.4.tgz", + "integrity": "sha512-p6WvbXOhfwnVtv/MyPq5kBiVhfV/qglMIxwwTVNEI1Ha1HmOgz5uHYHtYO6SfGckFaXe8blA11IJJ3wqXmvmBA==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.1.3", + "google-logging-utils": "^2.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=22" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -1198,6 +1305,62 @@ "node": ">= 6" } }, + "node_modules/google-auth-library": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-11.1.0.tgz", + "integrity": "sha512-YIBq5KNoHQrZ9PM4PeodDBqqBGlSTIxh1zibN8u3zNIO9UJg8gvKuCpdKV7fWXhsZ2zRkU6FhGrgKNgLJ2IRgw==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "^9.0.0", + "google-logging-utils": "^2.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/google-logging-utils": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-2.0.1.tgz", + "integrity": "sha512-HMhaQghlOTvbcb3c4T5jmmOMtG3JUF1iOQMezaJXL86CDS+Tm2vHd0IeLFRAx3+ewd+bo9E1HFHoy17X5aJa9A==", + "license": "Apache-2.0", + "engines": { + "node": ">=22" + } + }, + "node_modules/googleapis": { + "version": "182.0.0", + "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-182.0.0.tgz", + "integrity": "sha512-W8K/yy5kb1j6TiKEtSj4I4DgjeImVfSzYmdmp8Bdn3dHBamBjmpJUteF0R7QQ9Ym++2Yq+ZYXvIFBC7ywBohAA==", + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^11.0.0", + "googleapis-common": "^9.0.0" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/googleapis-common": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-9.1.0.tgz", + "integrity": "sha512-UJBElspYPJb0+fI8q0LxARw1hPnwd9yd5NqhKqEUPxjhnB1jdsRP7rlRTBp69JS5Vo10bCiRE9+ISsIG2A+Ttg==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "gaxios": "^7.3.0", + "google-auth-library": "^11.0.0", + "google-logging-utils": "^2.0.0", + "qs": "^6.7.0", + "url-template": "^2.0.8" + }, + "engines": { + "node": ">=22" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -1301,7 +1464,6 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "dev": true, "license": "MIT", "dependencies": { "agent-base": "^7.1.2", @@ -1315,7 +1477,6 @@ "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -1333,7 +1494,6 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/iconv-lite": { @@ -1457,6 +1617,15 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/jsonfile": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", @@ -2119,6 +2288,44 @@ "node": "^18 || ^20 || >= 21" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/node-gyp-build": { "version": "4.8.4", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", @@ -3246,6 +3453,12 @@ "node": ">= 0.8" } }, + "node_modules/url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==", + "license": "BSD" + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -3283,6 +3496,15 @@ "node": ">= 0.8" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", diff --git a/backend/package.json b/backend/package.json index 125b713..561475d 100644 --- a/backend/package.json +++ b/backend/package.json @@ -26,6 +26,7 @@ "express-rate-limit": "^8.7.0", "express-validator": "^7.2.1", "fs-extra": "^11.4.1", + "googleapis": "^182.0.0", "helmet": "^8.3.0", "jsonwebtoken": "^9.0.3", "mongoose": "^7.8.12", diff --git a/backend/src/config/env.js b/backend/src/config/env.js index 7c4bd04..20013c0 100644 --- a/backend/src/config/env.js +++ b/backend/src/config/env.js @@ -25,8 +25,11 @@ const schema = z.object({ ADMIN_FIRST_NAME: z.string().min(1).default('Admin'), ADMIN_SURNAME: z.string().min(1).default('Signpak'), - STORAGE_DRIVER: z.enum(['local']).default('local'), + STORAGE_DRIVER: z.enum(['local', 'gdrive']).default('local'), + ARCHIVE_STORAGE_DRIVER: z.enum(['local', 'gdrive']).default('local'), UPLOAD_DIR: z.string().default('uploads'), + GOOGLE_SERVICE_ACCOUNT_JSON: z.string().optional(), + GOOGLE_DRIVE_FOLDER_ID: z.string().optional(), MAX_VIDEO_UPLOAD_MB: z.coerce.number().positive().default(300), MAX_RECORDING_UPLOAD_MB: z.coerce.number().positive().default(100), @@ -36,6 +39,14 @@ const schema = z.object({ // minimum gap enforced between two submissions for the same (user, video). Overridable // (like BCRYPT_ROUNDS) so tests do not have to wait 30 real seconds. SUBMISSION_COOLDOWN_MS: z.coerce.number().int().positive().default(30_000), +}).superRefine((data, context) => { + const needsGoogle = data.STORAGE_DRIVER === 'gdrive' || data.ARCHIVE_STORAGE_DRIVER === 'gdrive' + if (needsGoogle && !data.GOOGLE_SERVICE_ACCOUNT_JSON) { + context.addIssue({ code: 'custom', path: ['GOOGLE_SERVICE_ACCOUNT_JSON'], message: 'Required when STORAGE_DRIVER=gdrive.' }) + } + if (needsGoogle && !data.GOOGLE_DRIVE_FOLDER_ID) { + context.addIssue({ code: 'custom', path: ['GOOGLE_DRIVE_FOLDER_ID'], message: 'Required when STORAGE_DRIVER=gdrive.' }) + } }) const parsed = schema.safeParse(process.env) diff --git a/backend/src/controllers/submissionController.js b/backend/src/controllers/submissionController.js index 75480fb..644580c 100644 --- a/backend/src/controllers/submissionController.js +++ b/backend/src/controllers/submissionController.js @@ -1,11 +1,8 @@ import { submissionService } from '../services/submissionService.js' import { asyncHandler } from '../utils/asyncHandler.js' import { created, ok } from '../utils/response.js' -import { sendFile } from '../utils/sendFile.js' export const submissionController = { create: asyncHandler(async (req, res) => created(res, await submissionService.create(req.user, req.body, req.file))), list: asyncHandler(async (req, res) => ok(res, await submissionService.list(req.user))), - // Recordings are private: no-store so browsers and proxies never keep a copy. - recording: asyncHandler(async (req, res) => sendFile(req, res, await submissionService.getRecording(req.params.id), { cache: 'private, no-store' })), } diff --git a/backend/src/docs/openapi.js b/backend/src/docs/openapi.js index 0da1afd..49f1603 100644 --- a/backend/src/docs/openapi.js +++ b/backend/src/docs/openapi.js @@ -133,7 +133,6 @@ export const openapi = { get: { tags: ['Submissions'], summary: 'List the current user submissions', security: userSecurity, responses: { 200: { description: 'Submissions' }, 401: errorResponse } }, post: { tags: ['Submissions'], summary: 'Submit a recording', security: userSecurity, requestBody: { required: true, content: { 'multipart/form-data': { schema: { type: 'object', required: ['recording', 'videoId'], properties: { recording: { type: 'string', format: 'binary' }, videoId: { type: 'string' }, trimStart: { type: 'number' }, trimEnd: { type: 'number' }, mirrored: { type: 'boolean' }, duration: { type: 'number' } } } } } }, responses: { 201: { description: 'Created' }, 400: errorResponse } }, }, - '/submissions/{id}/recording': { parameters: [idParameter], get: { tags: ['Submissions'], summary: 'Stream a submission recording', security: adminSecurity, responses: { 200: { description: 'Recording stream' }, 403: errorResponse } } }, '/contact': { post: { tags: ['Contact'], summary: 'Send a contact message', requestBody: { required: true, content: { 'application/json': { schema: { type: 'object', required: ['name', 'email', 'message'], properties: { name: { type: 'string' }, email: { type: 'string', format: 'email' }, message: { type: 'string' } } } } } }, responses: { 201: { description: 'Message sent' }, 400: errorResponse } } }, '/admin/users': { get: { tags: ['Admin'], summary: 'List all users', security: adminSecurity, responses: { 200: { description: 'Users' }, 403: errorResponse } } }, '/admin/stats': { get: { tags: ['Admin'], summary: 'Get dashboard statistics', security: adminSecurity, parameters: [{ name: 'days', in: 'query', schema: { type: 'integer', minimum: 7, maximum: 90, default: 14 } }], responses: { 200: { description: 'Statistics' }, 403: errorResponse } } }, diff --git a/backend/src/models/Submission.js b/backend/src/models/Submission.js index 3be4846..05ad542 100644 --- a/backend/src/models/Submission.js +++ b/backend/src/models/Submission.js @@ -18,6 +18,7 @@ const submissionSchema = new mongoose.Schema({ ret.submittedAt = ret.createdAt ret.size = ret.recording?.size ?? 0 ret.mimeType = ret.recording?.mimeType + ret.archivePath = ret.recording?.archivePath ?? null for (const key of ['user', 'video', 'recording', 'createdAt', 'updatedAt']) delete ret[key] return ret }), diff --git a/backend/src/models/storedFile.js b/backend/src/models/storedFile.js index 4aad584..320b2eb 100644 --- a/backend/src/models/storedFile.js +++ b/backend/src/models/storedFile.js @@ -10,4 +10,5 @@ export const storedFileSchema = new mongoose.Schema({ mimeType: { type: String, required: true }, size: { type: Number, required: true, min: 0 }, originalName: { type: String, default: '' }, + archivePath: { type: String, default: null }, }, { _id: false }) diff --git a/backend/src/repositories/submissionRepo.js b/backend/src/repositories/submissionRepo.js index 02de858..bc1f6d5 100644 --- a/backend/src/repositories/submissionRepo.js +++ b/backend/src/repositories/submissionRepo.js @@ -7,6 +7,7 @@ export const submissionRepo = { findById: (id) => Submission.findById(id), list: ({ userId } = {}) => Submission.find(userId ? { user: userId } : {}).sort({ createdAt: -1 }), recent: (limit) => Submission.find().sort({ createdAt: -1 }).limit(limit), + countForPair: (userId, videoId) => Submission.countDocuments({ user: userId, video: videoId }), count: () => Submission.countDocuments(), // Just the timestamps, for bucketing into days. createdSince: (date) => Submission.find({ createdAt: { $gte: date } }).select('createdAt').lean(), diff --git a/backend/src/routes/submissionRoutes.js b/backend/src/routes/submissionRoutes.js index 673c50f..2b1e46a 100644 --- a/backend/src/routes/submissionRoutes.js +++ b/backend/src/routes/submissionRoutes.js @@ -1,12 +1,9 @@ import { Router } from 'express' -import { ROLES } from '../config/constants.js' import { submissionController } from '../controllers/submissionController.js' import { authenticate } from '../middlewares/authenticate.js' import { uploadLimiter } from '../middlewares/rateLimiter.js' -import { requireRole } from '../middlewares/requireRole.js' import { recordingUpload } from '../middlewares/upload.js' import { validate } from '../middlewares/validate.js' -import { idParams } from '../middlewares/validators/common.js' import { createSubmissionSchema } from '../middlewares/validators/submissionSchemas.js' const router = Router() @@ -15,7 +12,6 @@ router.use(authenticate) router.get('/', submissionController.list) router.post('/', uploadLimiter, recordingUpload, validate({ body: createSubmissionSchema }), submissionController.create) -// No PATCH or DELETE exists on purpose: a submission cannot be changed or taken back. -router.get('/:id/recording', requireRole(ROLES.ADMIN), validate({ params: idParams }), submissionController.recording) +// No read, PATCH, or DELETE route exists: archived submissions are append-only. export default router diff --git a/backend/src/services/storage/googleDriveDriver.js b/backend/src/services/storage/googleDriveDriver.js new file mode 100644 index 0000000..87193a5 --- /dev/null +++ b/backend/src/services/storage/googleDriveDriver.js @@ -0,0 +1,83 @@ +import { createReadStream } from 'node:fs' +import { google } from 'googleapis' +import { randomUUID } from 'node:crypto' + +const DRIVE_SCOPE = 'https://www.googleapis.com/auth/drive' +const FOLDER_MIME = 'application/vnd.google-apps.folder' + +const escapeQueryValue = (value) => String(value).replace(/\\/g, '\\\\').replace(/'/g, "\\'") + +export function createGoogleDriveDriver({ serviceAccountJson, rootFolderId }) { + const credentials = JSON.parse(serviceAccountJson) + const auth = new google.auth.GoogleAuth({ credentials, scopes: [DRIVE_SCOPE] }) + const drive = google.drive({ version: 'v3', auth }) + const folderIds = new Map() + + async function folderIdFor(folderPath) { + let parentId = rootFolderId + let cacheKey = '' + for (const name of folderPath.split('/').filter(Boolean)) { + cacheKey = `${cacheKey}/${name}` + if (folderIds.has(cacheKey)) { + parentId = folderIds.get(cacheKey) + continue + } + const listed = await drive.files.list({ + q: `'${escapeQueryValue(parentId)}' in parents and name = '${escapeQueryValue(name)}' and mimeType = '${FOLDER_MIME}' and trashed = false`, + fields: 'files(id)', + pageSize: 1, + spaces: 'drive', + }) + let id = listed.data.files?.[0]?.id + if (!id) { + const created = await drive.files.create({ + requestBody: { name, mimeType: FOLDER_MIME, parents: [parentId] }, + fields: 'id', + }) + id = created.data.id + } + folderIds.set(cacheKey, id) + parentId = id + } + return parentId + } + + return { + name: 'gdrive', + tempDir: null, + + async init() { + await drive.files.get({ fileId: rootFolderId, fields: 'id, trashed' }) + }, + + async save({ tempPath, ext = '', folder, fileName, mimeType }) { + const parentId = await folderIdFor(folder) + const suffix = ext ? (ext.startsWith('.') ? ext : `.${ext}`) : '' + const uploaded = await drive.files.create({ + requestBody: { name: fileName || `${randomUUID()}${suffix}`, parents: [parentId], mimeType }, + media: { mimeType, body: createReadStream(tempPath) }, + fields: 'id', + }) + return { key: uploaded.data.id } + }, + + async stat(key) { + const response = await drive.files.get({ fileId: key, fields: 'size' }) + return { size: Number(response.data.size || 0) } + }, + + async createReadStream(key, range) { + const headers = range ? { Range: `bytes=${range.start}-${range.end}` } : undefined + const response = await drive.files.get({ fileId: key, alt: 'media' }, { responseType: 'stream', headers }) + return response.data + }, + + async remove(key) { + try { + await drive.files.delete({ fileId: key }) + } catch (error) { + if (error.code !== 404) throw error + } + }, + } +} diff --git a/backend/src/services/storage/index.js b/backend/src/services/storage/index.js index 46c6d33..45fa4f7 100644 --- a/backend/src/services/storage/index.js +++ b/backend/src/services/storage/index.js @@ -1,9 +1,14 @@ +import fs from 'node:fs/promises' import path from 'node:path' import { env } from '../../config/env.js' import { logger } from '../../utils/logger.js' +import { createGoogleDriveDriver } from './googleDriveDriver.js' import { createLocalDriver } from './localDriver.js' -const drivers = { local: createLocalDriver(path.resolve(env.UPLOAD_DIR)) } +const drivers = { + local: createLocalDriver(path.resolve(env.UPLOAD_DIR)), + ...(env.STORAGE_DRIVER === 'gdrive' || env.ARCHIVE_STORAGE_DRIVER === 'gdrive' ? { gdrive: createGoogleDriveDriver({ serviceAccountJson: env.GOOGLE_SERVICE_ACCOUNT_JSON, rootFolderId: env.GOOGLE_DRIVE_FOLDER_ID }) } : {}), +} const driverFor = (name) => { if (!drivers[name]) throw new Error(`Unknown storage driver "${name}"`) return drivers[name] @@ -20,11 +25,18 @@ export const storageService = { /** Takes a finished temp upload and returns the reference to store on a document. */ async save({ tempPath, originalName = '', mimeType, ext }, { folder }) { const driver = defaultDriver() - const { key } = await driver.save({ tempPath, ext, folder }) + const { key } = await driver.save({ tempPath, ext, folder, mimeType }) const { size } = await driver.stat(key) return { driver: driver.name, key, mimeType, size, originalName: originalName.slice(0, 200) } }, + async archive({ tempPath, originalName = '', mimeType, ext, folder, fileName }) { + const driver = driverFor(env.ARCHIVE_STORAGE_DRIVER) + const { size } = await fs.stat(tempPath) + const { key } = await driver.save({ tempPath, ext, folder, fileName, mimeType }) + return { driver: driver.name, key, mimeType, size, originalName: originalName.slice(0, 200), archivePath: `${folder}/${fileName || key}` } + }, + /** Descriptor consumed by utils/sendFile. */ async describe(ref) { const driver = driverFor(ref.driver) diff --git a/backend/src/services/storage/localDriver.js b/backend/src/services/storage/localDriver.js index e46e282..8a4f29b 100644 --- a/backend/src/services/storage/localDriver.js +++ b/backend/src/services/storage/localDriver.js @@ -34,8 +34,8 @@ export function createLocalDriver(rootDir) { await fs.mkdir(path.join(root, 'tmp'), { recursive: true }) }, - async save({ tempPath, ext = '', folder }) { - const key = `${folder}/${randomUUID()}${ext}` + async save({ tempPath, ext = '', folder, fileName }) { + const key = `${folder}/${fileName || `${randomUUID()}${ext}`}` const destination = resolveKey(key) await fs.mkdir(path.dirname(destination), { recursive: true }) try { diff --git a/backend/src/services/submissionService.js b/backend/src/services/submissionService.js index d223fe4..bcb118c 100644 --- a/backend/src/services/submissionService.js +++ b/backend/src/services/submissionService.js @@ -1,13 +1,14 @@ import { MAX_RECORDING_SECONDS, ROLES, TRIM_TOLERANCE_SECONDS, VIDEO_STATUS } from '../config/constants.js' import { env } from '../config/env.js' import { submissionRepo } from '../repositories/submissionRepo.js' +import { categoryRepo } from '../repositories/categoryRepo.js' import { videoRepo } from '../repositories/videoRepo.js' import { conflict, notFound, validationError } from '../utils/AppError.js' import { sniffVideo } from '../utils/files.js' -import { logger } from '../utils/logger.js' import { storageService } from './storage/index.js' const cooling = (remainingMs) => conflict(`Wait ${Math.ceil(remainingMs / 1000)}s before recording this video again.`) +const archiveSlug = (value) => String(value).normalize('NFKD').replace(/[\u0300-\u036f]/g, '').toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '').slice(0, 80) || 'untitled' function assertTrimIsSane({ trimStart, trimEnd, duration }) { if (duration <= 0 || duration > MAX_RECORDING_SECONDS) throw validationError({ duration: `A recording must be between 0 and ${MAX_RECORDING_SECONDS} seconds.` }) @@ -33,35 +34,40 @@ export const submissionService = { const video = await videoRepo.findById(input.videoId) if (!video || video.status !== VIDEO_STATUS.PUBLISHED || !video.category) throw notFound('That video is not available.') + const category = await categoryRepo.findById(video.category) + if (!category) throw notFound('That video category is not available.') const claimed = await submissionRepo.claimCooldown(user._id, video._id, env.SUBMISSION_COOLDOWN_MS) if (!claimed) throw cooling(await submissionRepo.cooldownRemaining(user._id, video._id)) - const recording = await storageService.save({ tempPath: upload.path, originalName: upload.originalname, mimeType: kind.mimeType, ext: kind.ext }, { folder: 'recordings' }) + const sequence = await submissionRepo.countForPair(user._id, video._id) + const categoryName = archiveSlug(category.label) + const videoName = archiveSlug(video.title) + const archiveFolder = `commons/${user._id}_${sequence}/${categoryName}/${videoName}` + const recording = await storageService.archive({ + tempPath: upload.path, originalName: upload.originalname, mimeType: kind.mimeType, ext: kind.ext, + folder: archiveFolder, fileName: `${videoName}${kind.ext}`, + }) try { return await submissionRepo.create({ user: user._id, video: video._id, trimStart: input.trimStart, trimEnd: input.trimEnd, mirrored: input.mirrored, duration: input.duration, recording, }) } catch (error) { - await storageService.remove(recording) await submissionRepo.releaseCooldown(user._id, video._id) // this attempt never landed, so it should not cost the contributor their cooldown throw error } }, /** Contributors see their own submissions; admins see all of them. */ - list: (user) => submissionRepo.list(user.role === ROLES.ADMIN ? {} : { userId: user._id }), - - /** Admin only (enforced by the route). Contributors can never play their own submission back. */ - async getRecording(id) { - const submission = await submissionRepo.findById(id) - if (!submission) throw notFound('That submission does not exist.') - try { - return await storageService.describe(submission.recording) - } catch (error) { - logger.error({ err: error, key: submission.recording.key }, 'Recording file could not be read') - throw notFound('That recording file is missing.') - } + async list(user) { + const isAdmin = user.role === ROLES.ADMIN + const submissions = await submissionRepo.list(isAdmin ? {} : { userId: user._id }) + return submissions.map((submission) => { + const result = submission.toJSON() + if (!isAdmin) delete result.archivePath + return result + }) }, + } diff --git a/backend/tests/api.test.js b/backend/tests/api.test.js index 42f3547..f9d3aa1 100644 --- a/backend/tests/api.test.js +++ b/backend/tests/api.test.js @@ -303,7 +303,7 @@ describe('submissions and the cooldown rule', () => { assert.equal(submission.mirrored, true) assert.equal(submission.trimEnd, 2.5) assert.equal(submission.recording, undefined) - assert.equal(filesIn('recordings').length, 1) + assert.equal(filesIn('commons').length, 1) const mine = await contributor.get('/submissions') assert.equal(mine.body.data.length, 1) assert.equal((await other.get('/submissions')).body.data.length, 0) @@ -314,14 +314,14 @@ describe('submissions and the cooldown rule', () => { const again = await contributor.upload('/submissions', recordingForm(good(), { file: mp4() })) assert.equal(again.status, 409) assert.match(again.body.error.message, /wait \d+s/i) - assert.equal(filesIn('recordings').length, 1) + assert.equal(filesIn('commons').length, 1) assert.deepEqual(filesIn('tmp'), []) }) test('COOLDOWN: concurrent double-submits for the same video produce exactly one submission', async () => { const results = await Promise.all([1, 2, 3].map(() => other.upload('/submissions', recordingForm({ ...good(), videoId: video2.id })))) assert.deepEqual(results.map((r) => r.status).sort(), [201, 409, 409]) - assert.equal(filesIn('recordings').length, 2, 'losing uploads must not leave orphaned files') + assert.equal(filesIn('commons').length, 2, 'losing uploads must not leave orphaned archive folders') assert.equal((await other.get('/submissions')).body.data.length, 1) }) @@ -334,20 +334,11 @@ describe('submissions and the cooldown rule', () => { assert.equal(mine.body.data.length, 2, 'both recordings for this video are kept, not merged or replaced') }) - test('a submission can never be edited or deleted once made, and contributors cannot watch their own back', async () => { + test('a submission can never be edited, deleted, or read once archived', async () => { assert.equal((await contributor.patch(`/submissions/${submission.id}`, { trimEnd: 1 })).status, 404) assert.equal((await contributor.delete(`/submissions/${submission.id}`)).status, 404) - assert.equal((await contributor.get(`/submissions/${submission.id}/recording`)).status, 403) - assert.equal((await other.get(`/submissions/${submission.id}/recording`)).status, 403) - }) - - test('admins can watch a submission (with Range), and it is not cacheable', async () => { - const res = await admin.get(`/submissions/${submission.id}/recording`, { raw: true }) - assert.equal(res.status, 200) - assert.deepEqual(res.body, webm(3000)) - assert.match(res.headers.get('cache-control'), /no-store/) - const part = await admin.get(`/submissions/${submission.id}/recording`, { raw: true, headers: { range: 'bytes=0-3' } }) - assert.equal(part.status, 206) + assert.equal((await contributor.get(`/submissions/${submission.id}/recording`)).status, 404) + assert.equal((await admin.get(`/submissions/${submission.id}/recording`)).status, 404) }) test('deleting a base video keeps contributors\' submissions on record', async () => { From 18da3c11517e7ce61de66709f55192a4bdf2af74 Mon Sep 17 00:00:00 2001 From: muhammadali-imran Date: Sat, 26 Sep 2026 03:09:47 +0500 Subject: [PATCH 2/5] backend automated testing --- .github/workflows/backend-tests.yml | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/backend-tests.yml diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml new file mode 100644 index 0000000..d2c45b5 --- /dev/null +++ b/.github/workflows/backend-tests.yml @@ -0,0 +1,33 @@ +name: Backend tests + +on: + push: + branches: [main, master] + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: backend/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run backend integration tests + run: npm test From 7a4935c9a9a69657d9f3b08b381870c121170c89 Mon Sep 17 00:00:00 2001 From: muhammadali-imran Date: Sat, 26 Sep 2026 03:16:32 +0500 Subject: [PATCH 3/5] setup.md --- backend/docs/setup.md | 239 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 backend/docs/setup.md diff --git a/backend/docs/setup.md b/backend/docs/setup.md new file mode 100644 index 0000000..37a8b27 --- /dev/null +++ b/backend/docs/setup.md @@ -0,0 +1,239 @@ +# Backend Setup + +This guide explains how to install, configure, run, test, and deploy the SignPak Commons backend. + +## 1. Prerequisites + +Install: + +- Node.js 20 or newer +- npm +- MongoDB 7+ locally, or a MongoDB Atlas cluster +- Git + +The backend does not require Google Drive for local development. Local storage is the default. + +## 2. Install the backend + +From the repository root: + +```bash +cd backend +npm install +``` + +Windows PowerShell uses the same commands: + +```powershell +Set-Location backend +npm install +``` + +## 3. Create the environment file + +Copy the committed template. Never commit the resulting `.env` file. + +macOS/Linux: + +```bash +cp .env.example .env +``` + +Windows PowerShell: + +```powershell +Copy-Item .env.example .env +``` + +The application loads `.env` automatically through `dotenv`. `src/config/env.js` validates every value at startup and stops the process with a readable error when a required value is missing or invalid. + +## 4. Configure MongoDB + +### Local MongoDB + +Start MongoDB and use: + +```env +MONGODB_URI=mongodb://127.0.0.1:27017/signpakcommons +``` + +### MongoDB Atlas + +1. Create a cluster at [MongoDB Atlas](https://www.mongodb.com/atlas). +2. Create a database user with access to the application database. +3. Add the backend server IP to Atlas Network Access. For local development, use your current IP rather than opening access to everyone. +4. Select **Connect > Drivers** and copy the Node.js connection string. +5. Replace the username, password, cluster host, and database name in `.env`: + +```env +MONGODB_URI=mongodb+srv://USERNAME:PASSWORD@CLUSTER.mongodb.net/signpakcommons?retryWrites=true&w=majority +``` + +URL-encode special characters in the database username or password. + +## 5. Environment variables + +### Runtime and networking + +| Variable | Where to get it | Purpose | +| --- | --- | --- | +| `NODE_ENV` | Set manually: `development`, `test`, or `production` | Selects runtime behavior and production cookie defaults. | +| `PORT` | Set manually; default `5000` | Port used by the Express server. | +| `LOG_LEVEL` | Set manually: `fatal`, `error`, `warn`, `info`, `debug`, `trace`, or `silent` | Controls structured log verbosity. | +| `TRUST_PROXY` | Set `true` behind Render, Railway, nginx, or another reverse proxy | Makes rate limiting use the original client IP. | +| `CLIENT_ORIGIN` | Frontend URL, for example `http://localhost:5173` | CORS allowlist and state-changing request origin check. Separate multiple origins with commas. | + +### Authentication and cookies + +| Variable | Where to get it | Purpose | +| --- | --- | --- | +| `JWT_SECRET` | Generate locally with the command below, or use a secret manager | Signs authentication tokens. Use a unique value of at least 32 characters. | +| `JWT_EXPIRES_DAYS` | Set manually; default `7` | Token lifetime in days. | +| `COOKIE_SECURE` | Usually leave unset; production defaults to `true` | Sends cookies only over HTTPS when true. Use false only for local HTTP development. | +| `COOKIE_SAMESITE` | Usually `lax`; use `none` only for cross-site HTTPS deployment | Controls cross-site cookie behavior. `none` requires `COOKIE_SECURE=true`. | +| `BCRYPT_ROUNDS` | Set manually from `4` to `15`; default `12` | Password hashing cost. Increase only when deployment CPU allows it. | + +Generate a JWT secret: + +```bash +node -e "console.log(require('crypto').randomBytes(48).toString('hex'))" +``` + +### Admin bootstrap + +| Variable | Where to get it | Purpose | +| --- | --- | --- | +| `ADMIN_EMAIL` | Choose the initial administrator email | Creates or promotes the admin account at startup. | +| `ADMIN_PASSWORD` | Choose a strong password, at least 8 characters and at most 72 | Sets the initial administrator password. | +| `ADMIN_FIRST_NAME` | Set manually; default `Admin` | Initial admin profile first name. | +| `ADMIN_SURNAME` | Set manually; default `Signpak` | Initial admin profile surname. | + +The admin is ensured on every startup. Changing `ADMIN_PASSWORD` later does not reset an existing admin password. + +### Storage + +| Variable | Where to get it | Purpose | +| --- | --- | --- | +| `STORAGE_DRIVER` | `local` or `gdrive` | Storage for readable base videos and posters. Use `local` for development. | +| `ARCHIVE_STORAGE_DRIVER` | `local` or `gdrive` | Append-only storage for contributor recordings. Use `gdrive` for production archives. | +| `UPLOAD_DIR` | Local filesystem path; default `uploads` | Temporary uploads and local storage root. | +| `GOOGLE_SERVICE_ACCOUNT_JSON` | Download from Google Cloud, described below | Service-account credentials used when either storage driver is `gdrive`. | +| `GOOGLE_DRIVE_FOLDER_ID` | The ID of the shared Google Drive root folder | Parent folder for application-created Drive folders. | +| `MAX_VIDEO_UPLOAD_MB` | Set manually; default `300` | Maximum base-video upload size. | +| `MAX_RECORDING_UPLOAD_MB` | Set manually; default `100` | Maximum contributor-recording upload size. | + +### Application controls + +| Variable | Where to get it | Purpose | +| --- | --- | --- | +| `RATE_LIMIT_ENABLED` | `true` or `false`; default `true` | Enables API and upload rate limits. Keep enabled outside local debugging. | +| `SUBMISSION_COOLDOWN_MS` | Set manually; default `30000` | Minimum delay between submissions for the same user and reference video. | + +## 6. Google Drive append-only archives + +Google Drive is optional for base videos and submission archives. When `ARCHIVE_STORAGE_DRIVER=gdrive`, contributor recordings are created under this structure: + +```text +{GOOGLE_DRIVE_FOLDER_ID}/ + commons/ + {userId}_{sequence}/ + {category_slug}/ + {video_slug}/ + {video_slug}.webm +``` + +A repeated submission receives the next sequence number instead of replacing an existing file. The API writes the file and stores its pointer and metadata in MongoDB. It does not expose a read, update, or delete route for archived recordings. + +### Create Google credentials + +1. Open [Google Cloud Console](https://console.cloud.google.com/). +2. Create or select a project. +3. Open **APIs & Services > Library** and enable **Google Drive API**. +4. Open **IAM & Admin > Service Accounts**. +5. Create a service account for the backend. +6. Create a JSON key under **Keys > Add key > Create new key > JSON**. +7. Store the downloaded JSON in a secret manager. Do not commit the JSON file. +8. Create a folder in Google Drive and copy its ID from the folder URL: + +```text +https://drive.google.com/drive/folders/FOLDER_ID +``` + +9. Share that folder with the service account email, usually shaped like `name@project.iam.gserviceaccount.com`. +10. Give the service account permission to create files and folders. +11. Put the complete JSON document into the deployment environment as one value: + +```env +ARCHIVE_STORAGE_DRIVER=gdrive +GOOGLE_DRIVE_FOLDER_ID=FOLDER_ID +GOOGLE_SERVICE_ACCOUNT_JSON={"type":"service_account",...} +``` + +For hosting platforms that provide a secret editor, paste the JSON there. For local development, keep it in `.env` only. Do not put the JSON in `public/`, source control, or a frontend environment variable. + +`STORAGE_DRIVER=gdrive` can also be used for base videos, but base videos need to remain readable by the public demo or authenticated library. Use `ARCHIVE_STORAGE_DRIVER=gdrive` when the intended behavior is write-only contributor archives. + +## 7. Start the backend + +Development mode restarts when source files change: + +```bash +npm run dev +``` + +Production-style start: + +```bash +npm start +``` + +The startup sequence is: + +1. Validate environment variables. +2. Connect to MongoDB. +3. Initialize the selected storage drivers. +4. Create or promote the configured admin account. +5. Start listening on `PORT`. + +Check the service: + +- API root: `http://localhost:5000/` +- Health: `http://localhost:5000/api/v1/health` +- Swagger UI: `http://localhost:5000/docs` +- OpenAPI JSON: `http://localhost:5000/docs.json` + +## 8. Run tests + +The backend test suite uses Node's built-in test runner and starts `mongodb-memory-server` when `MONGODB_URI_TEST` is not provided: + +```bash +npm test +``` + +To use a disposable MongoDB database instead, set `MONGODB_URI_TEST` before running tests. The test suite drops that database, so never point it at production data. + +The GitHub Actions workflow at `.github/workflows/backend-tests.yml` runs `npm ci` and `npm test` on pushes to `main` or `master` and on pull requests. + +## 9. Frontend connection + +The frontend uses the API base configured by `VITE_API_ORIGIN`. For local development, the Vite proxy can forward `/api` to the backend. Make sure: + +```env +CLIENT_ORIGIN=http://localhost:5173 +``` + +The browser must send credentials for the HTTP-only authentication cookie. In production, set `CLIENT_ORIGIN` to the exact deployed frontend origin and configure HTTPS cookie settings. + +## 10. Production checklist + +- Use MongoDB Atlas or a managed MongoDB deployment. +- Set `NODE_ENV=production`. +- Generate a unique `JWT_SECRET` and store it in the host secret manager. +- Set a strong admin password and never expose it to the frontend. +- Set the exact production frontend URL in `CLIENT_ORIGIN`. +- Use HTTPS with `COOKIE_SECURE=true`. +- Keep `COOKIE_SAMESITE=lax` when frontend and API share a site; use `none` only when required by cross-site deployment. +- Use `ARCHIVE_STORAGE_DRIVER=gdrive` with a restricted service account for append-only recordings. +- Keep `RATE_LIMIT_ENABLED=true`. +- Confirm the upload limits match available storage and request limits. +- Never commit `.env`, Google service-account JSON, JWT secrets, or database credentials. From d39ef63e28e0a42c76151f4ea2849e72d2ce4bbd Mon Sep 17 00:00:00 2001 From: muhammadali-imran Date: Sat, 26 Sep 2026 04:06:10 +0500 Subject: [PATCH 4/5] dependency error fix --- backend/package-lock.json | 116 +++++++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 2 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 7050c92..3302cbc 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -158,8 +158,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "license": "Python-2.0", - "peer": true + "license": "Python-2.0" }, "node_modules/array-flatten": { "version": "1.1.1", @@ -283,6 +282,7 @@ "integrity": "sha512-AIPKioV7/Y/8KfZ3AAhjPJxLLbY49S64Ym5DakZlUg75qQiTgUq9hEJoEwa4eUezPUlXRy/i5NpsKvo9jgKmoA==", "dev": true, "license": "Apache-2.0", + "peer": true, "peerDependencies": { "bare-abort-controller": "*" }, @@ -919,6 +919,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-4.22.3.tgz", "integrity": "sha512-Bdcs4+3qlpVlx2NRn6fgX2Ue2/gGRaPeawebgclM0ERSCqDpA+owF1fdPwjJUTAJWMTuAaxjDf+hzb0/4eKvvw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", @@ -1594,6 +1595,20 @@ "node": ">=0.12.0" } }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/js-yaml": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", @@ -1996,6 +2011,20 @@ "@types/webidl-conversions": "*" } }, + "node_modules/mongodb-memory-server-core/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/mongodb-memory-server-core/node_modules/bson": { "version": "6.10.4", "resolved": "https://registry.npmjs.org/bson/-/bson-6.10.4.tgz", @@ -2024,6 +2053,38 @@ } } }, + "node_modules/mongodb-memory-server-core/node_modules/gaxios": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.1.3.tgz", + "integrity": "sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-memory-server-core/node_modules/gaxios/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/mongodb-memory-server-core/node_modules/mongodb": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.21.0.tgz", @@ -2089,6 +2150,56 @@ "dev": true, "license": "MIT" }, + "node_modules/mongodb-memory-server-core/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/mongodb-memory-server-core/node_modules/node-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT", + "optional": true + }, + "node_modules/mongodb-memory-server-core/node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause", + "optional": true + }, + "node_modules/mongodb-memory-server-core/node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/mongodb-memory-server-core/node_modules/tr46": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", @@ -2552,6 +2663,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz", "integrity": "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", From 75b454faf259fb3d0044a120be460a809a0d3e86 Mon Sep 17 00:00:00 2001 From: muhammadali-imran Date: Sat, 26 Sep 2026 04:11:18 +0500 Subject: [PATCH 5/5] Fix backend CI npm lockfile compatibility --- .github/workflows/backend-tests.yml | 3 +++ backend/package-lock.json | 34 ++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index d2c45b5..e5a8b04 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -26,6 +26,9 @@ jobs: cache: npm cache-dependency-path: backend/package-lock.json + - name: Pin npm version + run: npm install --global npm@10 + - name: Install dependencies run: npm ci diff --git a/backend/package-lock.json b/backend/package-lock.json index 3302cbc..79d4ba4 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -158,7 +158,8 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "license": "Python-2.0" + "license": "Python-2.0", + "peer": true }, "node_modules/array-flatten": { "version": "1.1.1", @@ -282,7 +283,6 @@ "integrity": "sha512-AIPKioV7/Y/8KfZ3AAhjPJxLLbY49S64Ym5DakZlUg75qQiTgUq9hEJoEwa4eUezPUlXRy/i5NpsKvo9jgKmoA==", "dev": true, "license": "Apache-2.0", - "peer": true, "peerDependencies": { "bare-abort-controller": "*" }, @@ -919,7 +919,6 @@ "resolved": "https://registry.npmjs.org/express/-/express-4.22.3.tgz", "integrity": "sha512-Bdcs4+3qlpVlx2NRn6fgX2Ue2/gGRaPeawebgclM0ERSCqDpA+owF1fdPwjJUTAJWMTuAaxjDf+hzb0/4eKvvw==", "license": "MIT", - "peer": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", @@ -1602,6 +1601,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "engines": { "node": ">=8" }, @@ -2018,6 +2018,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "debug": "4" }, @@ -2060,6 +2061,7 @@ "dev": true, "license": "Apache-2.0", "optional": true, + "peer": true, "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^5.0.0", @@ -2077,6 +2079,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "agent-base": "6", "debug": "4" @@ -2085,6 +2088,22 @@ "node": ">= 6" } }, + "node_modules/mongodb-memory-server-core/node_modules/gcp-metadata": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.3.0.tgz", + "integrity": "sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "peer": true, + "dependencies": { + "gaxios": "^5.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/mongodb-memory-server-core/node_modules/mongodb": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.21.0.tgz", @@ -2157,6 +2176,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "whatwg-url": "^5.0.0" }, @@ -2178,7 +2198,8 @@ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "dev": true, "license": "MIT", - "optional": true + "optional": true, + "peer": true }, "node_modules/mongodb-memory-server-core/node_modules/node-fetch/node_modules/webidl-conversions": { "version": "3.0.1", @@ -2186,7 +2207,8 @@ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", "dev": true, "license": "BSD-2-Clause", - "optional": true + "optional": true, + "peer": true }, "node_modules/mongodb-memory-server-core/node_modules/node-fetch/node_modules/whatwg-url": { "version": "5.0.0", @@ -2195,6 +2217,7 @@ "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -2663,7 +2686,6 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz", "integrity": "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==", "license": "MIT", - "peer": true, "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0",