diff --git a/.gitignore b/.gitignore index a09ba3fd..087fed4d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ # Frontend: Node modules node_modules/ +# Python virtualenv used by the schema-doc tooling. Never commit it: a plain +# `git add -A` otherwise sweeps in ~2,700 files of binaries and .pyc. +.venv/ + # Frontend: Build output build/ dist/ @@ -26,3 +30,6 @@ pnpm-debug.log* # Generated/copied files public/data/**/* + +.tsbuildcache/node.tsbuildinfo +tsconfig.app.tsbuildinfo \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 0916d851..468fa897 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,6 +52,7 @@ "cross-env": "^10.1.0", "eslint": "^10.11.0", "eslint-plugin-react-hooks": "^7.1.1", + "exceljs": "^4.4.0", "globals": "^17.12.0", "jiti": "^2.7.0", "jsdom": "^30.1.1", @@ -955,6 +956,51 @@ } } }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/@fast-csv/parse/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@favware/skip-dependency": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@favware/skip-dependency/-/skip-dependency-1.2.2.tgz", @@ -3250,6 +3296,62 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/archiver": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.2.tgz", + "integrity": "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver-utils": "^2.1.0", + "async": "^3.2.4", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^2.2.0", + "zip-stream": "^4.1.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "dev": true, + "license": "MIT", + "dependencies": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/archiver/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", @@ -3297,6 +3399,13 @@ "node": ">=12" } }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "dev": true, + "license": "MIT" + }, "node_modules/autoprefixer": { "version": "10.6.1", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.6.1.tgz", @@ -3354,6 +3463,27 @@ "node": "18 || 20 || >=22" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/baseline-browser-mapping": { "version": "2.11.23", "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.23.tgz", @@ -3384,6 +3514,30 @@ "require-from-string": "^2.0.2" } }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "dev": true, + "license": "Unlicense", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -3396,6 +3550,40 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", + "dev": true, + "license": "MIT" + }, "node_modules/bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", @@ -3462,6 +3650,60 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/buffer-indexof-polyfill": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", + "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", + "dev": true, + "engines": { + "node": ">=0.2.0" + } + }, "node_modules/cacheable": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-2.5.0.tgz", @@ -3537,6 +3779,29 @@ "node": ">=18" } }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", + "dev": true, + "license": "MIT/X11", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, + "node_modules/chainsaw/node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "dev": true, + "license": "MIT/X11", + "engines": { + "node": "*" + } + }, "node_modules/chalk": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz", @@ -4194,6 +4459,44 @@ "dev": true, "license": "MIT" }, + "node_modules/compress-commons": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", + "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.2", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/compress-commons/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, "node_modules/config-chain": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", @@ -4349,6 +4652,48 @@ } } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", + "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", + "dev": true, + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/crc32-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -4880,6 +5225,13 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, + "node_modules/dayjs": { + "version": "1.11.23", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.23.tgz", + "integrity": "sha512-QDTCU0M0MxR3hQfnlDJfwekQiaanm1ubOD231u73WBckQ/fsamwRLiE2GBz6D3a/xF1NgfiDLJjXBa1hYOYTtQ==", + "dev": true, + "license": "MIT" + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -5065,6 +5417,16 @@ "dev": true, "license": "MIT" }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/enhanced-resolve": { "version": "5.24.2", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.24.2.tgz", @@ -5640,6 +6002,55 @@ "node": ">=0.10.0" } }, + "node_modules/exceljs": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/exceljs/-/exceljs-4.4.0.tgz", + "integrity": "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg==", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver": "^5.0.0", + "dayjs": "^1.8.34", + "fast-csv": "^4.3.1", + "jszip": "^3.10.1", + "readable-stream": "^3.6.0", + "saxes": "^5.0.1", + "tmp": "^0.2.0", + "unzipper": "^0.10.11", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=8.3.0" + } + }, + "node_modules/exceljs/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/exceljs/node_modules/saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "dev": true, + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/execa": { "version": "9.6.0", "resolved": "https://registry.npmjs.org/execa/-/execa-9.6.0.tgz", @@ -5717,6 +6128,20 @@ ], "license": "MIT" }, + "node_modules/fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -5906,6 +6331,13 @@ "readable-stream": "^2.0.0" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true, + "license": "MIT" + }, "node_modules/fs-extra": { "version": "11.3.2", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.2.tgz", @@ -5921,6 +6353,13 @@ "node": ">=14.14" } }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, + "license": "ISC" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -5935,6 +6374,23 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "deprecated": "This package is no longer supported.", + "dev": true, + "license": "ISC", + "dependencies": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "engines": { + "node": ">=0.6" + } + }, "node_modules/function-timeout": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/function-timeout/-/function-timeout-1.0.2.tgz", @@ -6009,6 +6465,28 @@ "traverse": "0.6.8" } }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -6022,6 +6500,37 @@ "node": ">=10.13.0" } }, + "node_modules/glob/node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/globals": { "version": "17.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-17.12.0.tgz", @@ -6295,6 +6804,27 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -6305,6 +6835,13 @@ "node": ">= 4" } }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "dev": true, + "license": "MIT" + }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", @@ -6380,6 +6917,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -6861,6 +7410,19 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jszip": { + "version": "3.10.2", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.2.tgz", + "integrity": "sha512-3l+rb15IOWtUhU0H5MFqES/T6Kh7abYwjosBey/vD6hDt8zoEffkSC5Ws5SGtgVw3gBx2NEbhTeSW1+kWkpyTQ==", + "dev": true, + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, "node_modules/keyv": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", @@ -6871,6 +7433,19 @@ "@keyv/serialize": "^1.1.1" } }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -6885,6 +7460,16 @@ "node": ">= 0.8.0" } }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", @@ -7153,6 +7738,13 @@ "dev": true, "license": "MIT" }, + "node_modules/listenercount": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", + "integrity": "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ==", + "dev": true, + "license": "ISC" + }, "node_modules/load-json-file": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", @@ -7226,6 +7818,20 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.escaperegexp": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", @@ -7233,6 +7839,49 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", @@ -7247,6 +7896,13 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.throttle": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", @@ -7254,6 +7910,20 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.uniqby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz", @@ -8079,6 +8749,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -8286,6 +8969,16 @@ "node": ">=12.20.0" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/onetime": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", @@ -8426,6 +9119,13 @@ "node": ">=4" } }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "dev": true, + "license": "(MIT AND Zlib)" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -8536,6 +9236,16 @@ "node": ">=8" } }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -9010,6 +9720,46 @@ "util-deprecate": "~1.0.1" } }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -9148,6 +9898,20 @@ "node": ">=4" } }, + "node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, "node_modules/robust-predicates": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", @@ -9421,6 +10185,13 @@ "dev": true, "license": "ISC" }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "dev": true, + "license": "MIT" + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -9888,6 +10659,38 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tar-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/temp-dir": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-3.0.0.tgz", @@ -10049,6 +10852,16 @@ "dev": true, "license": "MIT" }, + "node_modules/tmp": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -10442,6 +11255,25 @@ "node": ">= 10.0.0" } }, + "node_modules/unzipper": { + "version": "0.10.14", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.14.tgz", + "integrity": "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "big-integer": "^1.6.17", + "binary": "~0.3.0", + "bluebird": "~3.4.1", + "buffer-indexof-polyfill": "~1.0.0", + "duplexer2": "~0.1.4", + "fstream": "^1.0.12", + "graceful-fs": "^4.2.2", + "listenercount": "~1.0.1", + "readable-stream": "~2.3.6", + "setimmediate": "~1.0.4" + } + }, "node_modules/update-browserslist-db": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.3.tgz", @@ -10500,6 +11332,17 @@ "dev": true, "license": "MIT" }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "dev": true, + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -11134,6 +11977,13 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, "node_modules/xml-name-validator": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", @@ -11312,6 +12162,58 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zip-stream": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", + "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver-utils": "^3.0.4", + "compress-commons": "^4.1.2", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/zip-stream/node_modules/archiver-utils": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", + "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "glob": "^7.2.3", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/zip-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/zod": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", diff --git a/package.json b/package.json index 9bc5379a..62b2d87a 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "cross-env": "^10.1.0", "eslint": "^10.11.0", "eslint-plugin-react-hooks": "^7.1.1", + "exceljs": "^4.4.0", "globals": "^17.12.0", "jiti": "^2.7.0", "jsdom": "^30.1.1", diff --git a/scripts/build-timeseries-files.ts b/scripts/build-timeseries-files.ts index 71ab0fd6..50deb4f8 100644 --- a/scripts/build-timeseries-files.ts +++ b/scripts/build-timeseries-files.ts @@ -1,18 +1,22 @@ -// scripts/build-timeseries-index.ts -// Build a compact linkage index from arbitrary JSON under src/data/**. +// scripts/build-timeseries-files.ts +// Build a compact linkage index from arbitrary JSON under src/data/**, and the +// per-dataset files the site serves for plotting and download. // // Behavior (env flags): // - TS_INDEX_STRICT=1 : exit(1) if any timeseries is malformed (missing datasetId/id or pathwayID) // - TS_INDEX_DEBUG=1 : verbose logs for skipped files & reasons // // Detection & normalization: -// - Identify timeseries by $schema containing "pathwayTimeseries.v1.json" (or full URL). +// - Identify timeseries by $schema containing "pathwayTimeseries.v1.json" or +// "pathwayTimeseries.v2.json" (or the full URL). // - datasetId := datasetId | id // - pathwayIDs := pathwayID (string|array) | pathwayIds | pathwayId // - label := label | name // // Output: -// - Writes src/data/index.json with { byPathway, byDataset }. +// - src/data/index.gen.ts and public/data/index.json with { byPathway, byDataset }. +// - public/data//.json (a copy, fetched by the plots) and .csv +// (the download). import { promises as fs } from "node:fs"; import * as path from "node:path"; @@ -22,6 +26,7 @@ import { getTechnologyDefinition, UnknownTaxonomyError, } from "../src/utils/timeseriesTaxonomy.ts"; +import { jsonToCsv, sectorSegmentColumns } from "./timeseries-csv.ts"; const STRICT = process.env.TS_INDEX_STRICT === "1"; const DEBUG = process.env.TS_INDEX_DEBUG === "1"; @@ -62,11 +67,10 @@ type TimeseriesIndex = { const ROOT = process.cwd(); const DATA_DIR = path.join(ROOT, "src", "data"); -const OUT_FILE = path.join(DATA_DIR, "index.json"); const TS_SCHEMA_MATCHERS = [ "pathwayTimeseries.v1.json", - "http://pathways.rmi.org/schema/pathwayTimeseries.v1.json", + "pathwayTimeseries.v2.json", ]; function logDebug(...args: any[]) { @@ -208,6 +212,7 @@ async function main() { const sectors = new Set(); const geos = new Set(); const metrics = new Set(); + const segments = new Set(); for (const row of rawData as any[]) { rowCount++; @@ -236,6 +241,10 @@ async function main() { // Unique metrics if (typeof row?.metric === "string" && row.metric) metrics.add(row.metric); + // Unique sector segments (v2 rows carry a list) + if (Array.isArray(row?.sectorSegment)) + for (const seg of row.sectorSegment) + if (typeof seg === "string" && seg) segments.add(seg); } if (rowCount > 0) computed.rowCount = rowCount; @@ -250,6 +259,7 @@ async function main() { computed.geographyCount = geos.size; computed.geographies = [...geos].sort(); } + if (segments.size > 0) computed.sectorSegments = [...segments].sort(); if (metrics.size > 0) computed.metrics = [...metrics].sort(); } @@ -307,21 +317,6 @@ async function main() { const PUBLIC_DATA_DIR = path.join(ROOT, "public", "data"); await fs.mkdir(PUBLIC_DATA_DIR, { recursive: true }); - function jsonToCsv(data: any[]): string { - if (!Array.isArray(data) || data.length === 0) return ""; - // Merge metadata into each row - const keys = Array.from(new Set(data.flatMap((row) => Object.keys(row)))); - const escape = (v: any) => - typeof v === "string" - ? `"${v.replace(/"/g, '""')}"` - : v === null || v === undefined - ? "" - : String(v); - const header = keys.join(","); - const rows = data.map((row) => keys.map((k) => escape(row[k])).join(",")); - return [header, ...rows].join("\n"); - } - for (const [dsId, ds] of Object.entries(out.byDataset)) { logDebug(`Processing datasetId: ${dsId}`); // ds.path currently looks like "/data/…file.json" (derived from src earlier). @@ -390,7 +385,6 @@ async function main() { source, sector: sectorDef.displayName, emissionsScope: parsed.emissionsScope, - sectorScope: metricDef?.sectorScope ?? "", metric: metricDef?.displayName ?? "", definitionMetric: metricDef?.definition ?? "", technology: techDef?.displayName ?? "", @@ -434,6 +428,12 @@ async function main() { : String(row.technology); const rowMetadata = getRowMetadata(sector, metric, technology); + const segmentColumns = sectorSegmentColumns( + sector, + Array.isArray(row.sectorSegment) + ? (row.sectorSegment as string[]) + : undefined, + ); const out = { publisher: rowMetadata.publisher, @@ -449,9 +449,11 @@ async function main() { unit: row.unit, source: rowMetadata.source, emissions_scope: rowMetadata.emissionsScope, - sector_scope: rowMetadata.sectorScope, + sector_segment: segmentColumns.sector_segment, definition_technology: rowMetadata.definitionTechnology, definition_metric: rowMetadata.definitionMetric, + definition_sector_segment: + segmentColumns.definition_sector_segment, }; logDebug(` Export row: ${JSON.stringify(out)}`); return out; diff --git a/scripts/codemod-timeseries-v1-to-v2.test.ts b/scripts/codemod-timeseries-v1-to-v2.test.ts new file mode 100644 index 00000000..3d284955 --- /dev/null +++ b/scripts/codemod-timeseries-v1-to-v2.test.ts @@ -0,0 +1,130 @@ +import { describe, it, expect } from "vitest"; +import { + editText, + relabel, + upgradeTimeseries, +} from "./codemod-timeseries-v1-to-v2.ts"; +import type { PathwayMetadataV2 } from "../src/types/pathwayMetadata.v2.d.ts"; + +type Geography = PathwayMetadataV2["geography"]; + +const geographyById = new Map([ + ["IEA-X", { global: true, regions: { "Southeast Asia": ["TH", "VN"] } }], + ["ACE-X", { regions: { ASEAN: ["TH", "VN"] } }], +]); +const renames = { + "IEA-X": { "South East Asia": "Southeast Asia" }, + "ACE-X": { "South East Asia": "ASEAN" }, +}; + +describe("relabel", () => { + it("keeps a label the pathway declares", () => { + expect(relabel("TH", ["IEA-X"], geographyById, renames)).toBe("TH"); + }); + + it("renames by the pathway's own entry, so the same label can map differently", () => { + expect(relabel("South East Asia", ["IEA-X"], geographyById, renames)).toBe( + "Southeast Asia", + ); + expect(relabel("South East Asia", ["ACE-X"], geographyById, renames)).toBe( + "ASEAN", + ); + }); + + it("refuses a label with no rename rather than guessing", () => { + expect(() => + relabel("Asia Pacific", ["IEA-X"], geographyById, renames), + ).toThrow(/not declared by IEA-X/); + }); + + it("refuses when the pathways a file serves disagree", () => { + expect(() => + relabel("South East Asia", ["IEA-X", "ACE-X"], geographyById, renames), + ).toThrow(); + }); +}); + +const V1 = "http://pathways.rmi.org/schema/pathwayTimeseries.v1.json"; +const v1Text = `{ + "$schema": "${V1}", + "pathwayId": ["IEA-X"], + "data": [ + { + "year": 2030, + "geography": "South East Asia", + "sector": "power", + "technology": null, + "metric": "capacity", + "value": 12.5, + "unit": "GW" + } + ] +} +`; + +describe("upgradeTimeseries + editText", () => { + it("changes only $schema, the label and sectorSegment, and the text edit agrees", () => { + const { doc, renamed } = upgradeTimeseries( + JSON.parse(v1Text) as Record, + new Map([["IEA-X", geographyById.get("IEA-X")]]), + renames, + ); + expect(doc.$schema).toBe( + "http://pathways.rmi.org/schema/pathwayTimeseries.v2.json", + ); + expect(renamed).toEqual(new Map([["South East Asia", "Southeast Asia"]])); + + const edited = editText(v1Text, renamed); + expect(JSON.parse(edited)).toEqual(doc); + // Every other line survives byte for byte. + expect(edited.split("\n")).toHaveLength(v1Text.split("\n").length + 1); + expect(edited).toContain(' "value": 12.5,'); + }); + + it("puts sectorSegment right after sector", () => { + const { doc } = upgradeTimeseries( + JSON.parse(v1Text) as Record, + new Map([["IEA-X", geographyById.get("IEA-X")]]), + renames, + ); + const row = (doc.data as Record[])[0]; + expect(Object.keys(row).slice(2, 4)).toEqual(["sector", "sectorSegment"]); + expect(row.sectorSegment).toEqual(["Power generation"]); + }); + + it("refuses a non-Power row rather than guess its segment", () => { + const doc = JSON.parse(v1Text.replace('"power"', '"steel"')) as Record< + string, + unknown + >; + expect(() => + upgradeTimeseries(doc, new Map([["IEA-X", geographyById.get("IEA-X")]])), + ).toThrow(/no segment known/); + }); + + it.each([ + [[], /pathwayId is empty/], + [["IEA-X", "IEA-X"], /lists an id twice/], + ])("refuses a pathwayId list v2 rejects: %j", (pathwayId, message) => { + const doc = { + ...(JSON.parse(v1Text) as Record), + pathwayId, + }; + expect(() => + upgradeTimeseries( + doc, + new Map([["IEA-X", geographyById.get("IEA-X")]]), + renames, + ), + ).toThrow(message); + }); + + it("refuses a file whose pathway has no metadata", () => { + expect(() => + upgradeTimeseries( + JSON.parse(v1Text) as Record, + new Map(), + ), + ).toThrow(/no metadata for IEA-X/); + }); +}); diff --git a/scripts/codemod-timeseries-v1-to-v2.ts b/scripts/codemod-timeseries-v1-to-v2.ts new file mode 100644 index 00000000..af863627 --- /dev/null +++ b/scripts/codemod-timeseries-v1-to-v2.ts @@ -0,0 +1,193 @@ +/** + * One-off migration of timeseries files from pathwayTimeseries.v1 to v2 (#915). + * + * Usage: + * node --experimental-strip-types scripts/codemod-timeseries-v1-to-v2.ts [--dry-run] + * + * Per file it changes three things and nothing else: + * + * - `$schema` becomes the v2 id. + * - Every row gains `sectorSegment: ["Power generation"]`. That is exactly what + * v1 meant: the taxonomy gave every Power metric the sector scope "Power + * generation", and every row in the corpus is Power. The codemod refuses any + * other sector rather than guess its segment. + * - A row's `geography` becomes the label its pathway's metadata declares. + * Labels already declared are kept. The rest come from {@link RENAMES}, an + * explicit table rather than a fuzzy match: "South East Asia" means "ASEAN" + * for ACE but "Southeast Asia" for IEA, which no spelling rule can know. A + * label neither declared nor in the table stops the run. + * + * Values are never touched. The file is edited line by line, so the diff shows + * only the changed lines, and the edited text is then parsed back and compared + * with the intended document, so a formatting surprise cannot slip through. + */ +import { promises as fs } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { declaredGeographies } from "../src/utils/validateScopes.ts"; +import type { PathwayMetadataV2 } from "../src/types/pathwayMetadata.v2.d.ts"; + +const V1_ID = "http://pathways.rmi.org/schema/pathwayTimeseries.v1.json"; +const V2_ID = "http://pathways.rmi.org/schema/pathwayTimeseries.v2.json"; +const METADATA_IDS = new Set([ + "http://pathways.rmi.org/schema/pathwayMetadata.v1.json", + "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", +]); +const POWER_SEGMENT = "Power generation"; + +type Geography = PathwayMetadataV2["geography"]; +type Json = Record; +type Row = Json & { geography: string; sector: string }; + +/** + * Timeseries labels that differ from their pathway's metadata, keyed by + * pathway id. Reviewed by hand: each says which region of the publication the + * series is. + */ +export const RENAMES: Record> = { + // ACE's 8th ASEAN Energy Outlook models the ASEAN member states. + "ACE-ATS-2024": { "South East Asia": "ASEAN" }, + "ACE-BAS-2024": { "South East Asia": "ASEAN" }, + "ACE-CNS-2024": { "South East Asia": "ASEAN" }, + "ACE-RAS-2024": { "South East Asia": "ASEAN" }, + // IEA's WEO spells its region "Southeast Asia" (#945). + "IEA-APS-2024": { "South East Asia": "Southeast Asia" }, + "IEA-STEPS-2024": { "South East Asia": "Southeast Asia" }, +}; + +/** + * The label a row's geography becomes: kept when every pathway declares it, + * renamed when every pathway's {@link RENAMES} entry agrees and declares the + * result. Throws otherwise. + */ +export function relabel( + token: string, + pathwayIds: readonly string[], + geographyById: ReadonlyMap, + renames: Record> = RENAMES, +): string { + const declaredByAll = (label: string) => + pathwayIds.every((id) => + declaredGeographies(geographyById.get(id)).has(label), + ); + if (declaredByAll(token)) return token; + const targets = new Set(pathwayIds.map((id) => renames[id]?.[token])); + const [target] = targets; + if (targets.size === 1 && target !== undefined && declaredByAll(target)) + return target; + throw new Error( + `geography "${token}" is not declared by ${pathwayIds.join(", ")} and has no agreed rename to a declared label`, + ); +} + +/** The v2 document, as plain data. Pure; does no I/O. */ +export function upgradeTimeseries( + doc: Json, + geographyById: ReadonlyMap, + renames: Record> = RENAMES, +): { doc: Json; renamed: Map } { + if (doc.$schema !== V1_ID) throw new Error(`not a v1 timeseries file`); + const pathwayIds = doc.pathwayId as string[]; + // v1 allowed both; v2 rejects them, and an empty list would make every + // geography look declared by "all" of no pathways. + if (pathwayIds.length === 0) throw new Error(`pathwayId is empty`); + if (new Set(pathwayIds).size !== pathwayIds.length) + throw new Error(`pathwayId lists an id twice: ${pathwayIds.join(", ")}`); + const missing = pathwayIds.filter((id) => !geographyById.has(id)); + if (missing.length) throw new Error(`no metadata for ${missing.join(", ")}`); + + const renamed = new Map(); + const data = (doc.data as Row[]).map((row) => { + if (row.sector !== "power") + throw new Error(`sector "${row.sector}": no segment known to assign`); + const geography = relabel( + row.geography, + pathwayIds, + geographyById, + renames, + ); + if (geography !== row.geography) renamed.set(row.geography, geography); + // Same key order the text edit produces: sectorSegment right after sector. + const out: Json = {}; + for (const [k, v] of Object.entries(row)) { + out[k] = k === "geography" ? geography : v; + if (k === "sector") out.sectorSegment = [POWER_SEGMENT]; + } + return out; + }); + return { doc: { ...doc, $schema: V2_ID, data }, renamed }; +} + +/** + * Apply the upgrade to the file's text line by line, so formatting and every + * untouched line survive byte for byte. + */ +export function editText(text: string, renamed: Map): string { + return text + .split("\n") + .flatMap((line) => { + if (line.includes(`"$schema": "${V1_ID}"`)) + return [line.replace(V1_ID, V2_ID)]; + const geo = /^(\s*)"geography": "(.*)",$/.exec(line); + if (geo && renamed.has(geo[2])) + return [ + `${geo[1]}"geography": ${JSON.stringify(renamed.get(geo[2]))},`, + ]; + const sector = /^(\s*)"sector": "power",$/.exec(line); + if (sector) + return [line, `${sector[1]}"sectorSegment": ["${POWER_SEGMENT}"],`]; + return [line]; + }) + .join("\n"); +} + +async function jsonFilesUnder(dir: string): Promise { + const out: string[] = []; + for (const d of await fs.readdir(dir, { withFileTypes: true })) { + const full = join(dir, d.name); + if (d.isDirectory()) out.push(...(await jsonFilesUnder(full))); + else if (d.name.endsWith(".json")) out.push(full); + } + return out; +} + +async function main() { + const dryRun = process.argv.includes("--dry-run"); + const files = await jsonFilesUnder("src/data"); + const docs = await Promise.all( + files.map(async (file) => { + const text = await fs.readFile(file, "utf8"); + return { file, text, doc: JSON.parse(text) as Json }; + }), + ); + + const geographyById = new Map( + docs + .filter(({ doc }) => METADATA_IDS.has(String(doc.$schema))) + .map(({ doc }) => [String(doc.id), doc.geography as Geography]), + ); + + let migrated = 0; + for (const { file, text, doc } of docs) { + if (doc.$schema !== V1_ID) continue; + const { doc: next, renamed } = upgradeTimeseries(doc, geographyById); + const edited = editText(text, renamed); + if (JSON.stringify(JSON.parse(edited)) !== JSON.stringify(next)) + throw new Error(`${file}: line edit does not reproduce the v2 document`); + const renames = [...renamed].map(([a, b]) => `"${a}" -> "${b}"`); + console.info( + `${dryRun ? "would migrate" : "migrated"} ${file}` + + (renames.length ? ` (${renames.join(", ")})` : ""), + ); + if (!dryRun) await fs.writeFile(file, edited); + migrated++; + } + console.info(`${migrated} file(s). Run \`npm run schema:check\` next.`); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main().catch((e: unknown) => { + console.error(String(e instanceof Error ? e.message : e)); + process.exit(1); + }); +} diff --git a/scripts/codemod-v1-to-v2.test.ts b/scripts/codemod-v1-to-v2.test.ts new file mode 100644 index 00000000..0b03e2e4 --- /dev/null +++ b/scripts/codemod-v1-to-v2.test.ts @@ -0,0 +1,346 @@ +import { describe, it, expect } from "vitest"; +import { + splitExpertOverview, + widestScope, + upgradeV1ToV2, + PATHWAY_DESCRIPTION, + CORE_DRIVERS, + TRANSITION_ASSESSMENT, +} from "./codemod-v1-to-v2.ts"; +import type { PathwayMetadataV1 } from "../src/types/pathwayMetadata.v1.d.ts"; + +const V2_ID = "http://pathways.rmi.org/schema/pathwayMetadata.v2.json"; + +const WELL_FORMED = [ + "#### Pathway Description", + "", + "A description of the pathway.", + "", + "#### Core Drivers", + "", + "*Policy:* Policies drive it.", + "", + "#### Application to Transition Assessment", + "", + "How to apply it.", +].join("\n"); + +/** ACE-CNS-2024's shape: the Core Drivers heading lost its `####` markers. */ +const BARE_HEADING = [ + "#### Pathway Description", + "", + "A description of the pathway.", + "", + "Core Drivers", + "", + "*Policy:* Policies drive it.", + "", + "#### Application to Transition Assessment", + "", + "How to apply it.", +].join("\n"); + +describe("splitExpertOverview", () => { + it("splits the three ATX-headed sections", () => { + const s = splitExpertOverview(WELL_FORMED); + expect(s.get(PATHWAY_DESCRIPTION)).toBe("A description of the pathway."); + expect(s.get(CORE_DRIVERS)).toBe("*Policy:* Policies drive it."); + expect(s.get(TRANSITION_ASSESSMENT)).toBe("How to apply it."); + }); + + it("treats a bare section title as a heading (the ACE-CNS case)", () => { + const s = splitExpertOverview(BARE_HEADING); + // Without the tolerance, Core Drivers prose would land in the description + // and push it over the 2500-char limit. + expect(s.get(PATHWAY_DESCRIPTION)).toBe("A description of the pathway."); + expect(s.get(CORE_DRIVERS)).toBe("*Policy:* Policies drive it."); + }); + + it("does not treat a title mentioned mid-sentence as a heading", () => { + const s = splitExpertOverview( + [ + "#### Pathway Description", + "", + "The Core Drivers of this pathway are policy-led.", + ].join("\n"), + ); + expect(s.get(PATHWAY_DESCRIPTION)).toBe( + "The Core Drivers of this pathway are policy-led.", + ); + expect(s.has(CORE_DRIVERS)).toBe(false); + }); + + it("preserves markdown inside a section body", () => { + const s = splitExpertOverview( + ["#### Pathway Description", "", "para one.", "", "para two."].join("\n"), + ); + expect(s.get(PATHWAY_DESCRIPTION)).toBe("para one.\n\npara two."); + }); + + it("returns no sections for text with no recognised headings", () => { + expect(splitExpertOverview("Just prose.").size).toBe(0); + }); +}); + +function v1(over: Partial): PathwayMetadataV1 { + return { + $schema: "http://pathways.rmi.org/schema/pathwayMetadata.v1.json", + id: "X", + name: { full: "X" }, + description: "A pathway.", + publication: { + title: { full: "T" }, + publisher: { full: "TransitionZero" }, + year: 2024, + }, + pathwayType: "Normative", + geography: { global: true }, + sectors: [{ name: "Power", technologies: [] }], + expertOverview: WELL_FORMED, + metric: ["Capacity"], + keyFeatures: { + emissionsTrajectory: "Significant decrease", + energyEfficiency: "Moderate improvement", + energyDemand: "Minor increase", + electrification: "Significant increase", + policyTypes: ["Carbon price"], + technologyCostTrend: "Decrease", + emissionsScope: "CO2e (Kyoto)", + policyAmbition: "High ambition policies", + technologyCostsDetail: "Total costs", + newTechnologiesIncluded: ["CCUS"], + investmentNeeds: "By sector", + }, + ...over, + } as unknown as PathwayMetadataV1; +} + +describe("widestScope", () => { + it("uses the lone sector when there is only one", () => { + expect(widestScope(v1({})).sector).toBe("Power"); + }); + + it("uses across sectors for a multi-sector pathway", () => { + const doc = v1({ + sectors: [ + { name: "Power", technologies: [] }, + { name: "Steel", technologies: [] }, + ], + } as Partial); + expect(widestScope(doc).sector).toBe("across sectors"); + }); + + it("collapses a repeated sector name rather than calling it multi-sector", () => { + // pathwayMetadata has no uniqueItems on sectors, and the _full fixture + // deliberately lists Automotive twice. + const doc = v1({ + sectors: [ + { name: "Power", technologies: [] }, + { name: "Power", technologies: ["Solar"] }, + ], + } as Partial); + expect(widestScope(doc).sector).toBe("Power"); + }); + + it("prefers Global when the pathway is global, even with regions present", () => { + const doc = v1({ + geography: { global: true, regions: { "South East Asia": ["TH"] } }, + } as Partial); + expect(widestScope(doc).geography).toBe("Global"); + }); + + it("uses the lone region label (the ACE case)", () => { + const doc = v1({ + geography: { regions: { "South East Asia": ["TH", "VN"] } }, + } as Partial); + expect(widestScope(doc).geography).toBe("South East Asia"); + }); + + it("uses the lone country code", () => { + const doc = v1({ + geography: { country: ["TH"] }, + } as Partial); + expect(widestScope(doc).geography).toBe("TH"); + }); + + it("throws rather than guessing between several regions", () => { + const doc = v1({ + geography: { regions: { A: ["TH"], B: ["VN"] } }, + } as Partial); + expect(() => widestScope(doc)).toThrow(/ambiguous/); + }); + + it("throws rather than guessing between several countries", () => { + const doc = v1({ + geography: { country: ["TH", "VN"] }, + } as Partial); + expect(() => widestScope(doc)).toThrow(/ambiguous/); + }); + + it("throws when there is no geography at all", () => { + const doc = v1({ geography: {} } as Partial); + expect(() => widestScope(doc)).toThrow(/no geography/); + }); +}); + +describe("upgradeV1ToV2", () => { + it("repoints $schema at v2", () => { + expect(upgradeV1ToV2(v1({})).doc.$schema).toBe(V2_ID); + }); + + it("wraps every keyFeature as one entry at the widest scope", () => { + const { doc } = upgradeV1ToV2(v1({})); + const kf = doc.keyFeatures; + expect(Object.keys(kf)).toHaveLength(11); + for (const entries of Object.values(kf)) { + expect(entries).toHaveLength(1); + expect(entries[0].sector).toBe("Power"); + expect(entries[0].geography).toBe("Global"); + } + expect(kf.emissionsTrajectory[0].value).toBe("Significant decrease"); + }); + + it("nests an array-valued field rather than splatting it into entries", () => { + const doc = upgradeV1ToV2( + v1({ + keyFeatures: { + ...v1({}).keyFeatures, + policyTypes: ["Carbon price", "Subsidies"], + }, + } as Partial), + ).doc; + expect(doc.keyFeatures.policyTypes).toHaveLength(1); + expect(doc.keyFeatures.policyTypes[0].value).toEqual([ + "Carbon price", + "Subsidies", + ]); + }); + + it("moves the description section into pathwayDescription", () => { + const { doc } = upgradeV1ToV2(v1({})); + expect(doc.pathwayDescription).toBe("A description of the pathway."); + // The assessment section has no v2 field; it comes back in the result so the + // run report can print it rather than losing it. + expect("transitionAssessment" in doc).toBe(false); + }); + + it("discards pathwayOverview rather than folding it in", () => { + // #858 says to fold it in; the data owner confirmed on PR #898 that it should + // be dropped, because the two texts restate each other and the merged field + // reads as immediate self-repetition. + const { doc, droppedPathwayOverview } = upgradeV1ToV2( + v1({ pathwayOverview: "A short summary." } as Partial), + ); + expect(droppedPathwayOverview).toBe("A short summary.".length); + expect(doc.pathwayDescription).toBe("A description of the pathway."); + expect(doc.pathwayDescription).not.toContain("A short summary."); + }); + + it("reports nothing dropped when there was no pathwayOverview", () => { + expect(upgradeV1ToV2(v1({})).droppedPathwayOverview).toBe(0); + }); + + it("drops the v1 overview fields", () => { + const { doc } = upgradeV1ToV2( + v1({ pathwayOverview: "A short summary." } as Partial), + ); + expect("expertOverview" in doc).toBe(false); + expect("pathwayOverview" in doc).toBe(false); + }); + + it("scaffolds coreDrivers all-null and dependencies empty", () => { + const { doc } = upgradeV1ToV2(v1({})); + expect(Object.values(doc.coreDrivers)).toEqual([ + null, + null, + null, + null, + null, + null, + null, + ]); + expect(doc.dependencies).toEqual([]); + }); + + it("returns the retired Application-to-Transition-Assessment prose", () => { + expect(upgradeV1ToV2(v1({})).transitionAssessmentProse).toBe( + "How to apply it.", + ); + }); + + it("returns the Core Drivers prose it does not carry over", () => { + expect(upgradeV1ToV2(v1({})).coreDriversProse).toBe( + "*Policy:* Policies drive it.", + ); + }); + + it("nulls pathwayDescription when there is no description section", () => { + const { doc } = upgradeV1ToV2( + v1({ expertOverview: "No headings here." } as Partial), + ); + expect(doc.pathwayDescription).toBeNull(); + }); + + it("preserves unrelated fields and their order", () => { + const { doc } = upgradeV1ToV2( + v1({ modelYearNetzero: 2050 } as Partial), + ); + expect(doc.modelYearNetzero).toBe(2050); + // pathwayDescription takes expertOverview's slot; coreDrivers/dependencies + // follow keyFeatures. Key order keeps the data-file diffs readable. + const keys = Object.keys(doc); + expect(keys.indexOf("pathwayDescription")).toBeLessThan( + keys.indexOf("metric"), + ); + expect(keys.indexOf("coreDrivers")).toBeGreaterThan( + keys.indexOf("keyFeatures"), + ); + }); +}); + +describe("upgradeV1ToV2 — #461 technology reporting", () => { + it("reports nothing when every technology belongs to its sector", () => { + const { unscopedTechnologies } = upgradeV1ToV2( + v1({ + sectors: [{ name: "Power", technologies: ["Solar", "Wind"] }], + } as unknown as Partial), + ); + expect(unscopedTechnologies).toEqual([]); + }); + + it("reports a technology outside its sector's list", () => { + const { unscopedTechnologies } = upgradeV1ToV2( + v1({ + sectors: [{ name: "Power", technologies: ["Solar", "Hydrogen Use"] }], + } as unknown as Partial), + ); + expect(unscopedTechnologies).toEqual(["Power: Hydrogen Use"]); + }); + + it("reports technologies on a sector with no defined list", () => { + const { unscopedTechnologies } = upgradeV1ToV2( + v1({ + sectors: [{ name: "Steel", technologies: ["Hydrogen Use"] }], + } as unknown as Partial), + ); + expect(unscopedTechnologies).toEqual(["Steel: Hydrogen Use"]); + }); + + it("leaves the offending technologies in the output document", () => { + // Reporting, not fixing: deleting a technology someone recorded on purpose + // would be worse than a failing schema:check with a line in the report. + const { doc, unscopedTechnologies } = upgradeV1ToV2( + v1({ + sectors: [{ name: "Steel", technologies: ["Hydrogen Use"] }], + } as unknown as Partial), + ); + expect(unscopedTechnologies).toHaveLength(1); + expect(doc.sectors).toEqual([ + { name: "Steel", technologies: ["Hydrogen Use"] }, + ]); + }); + + it("is empty for the corpus shape — TransitionZero declares no technologies", () => { + expect(upgradeV1ToV2(v1({})).unscopedTechnologies).toEqual([]); + }); +}); diff --git a/scripts/codemod-v1-to-v2.ts b/scripts/codemod-v1-to-v2.ts new file mode 100644 index 00000000..575b0659 --- /dev/null +++ b/scripts/codemod-v1-to-v2.ts @@ -0,0 +1,393 @@ +/** + * One-shot codemod: pathwayMetadata v1 -> v2 (#858). + * + * Run over an explicit list of files, in place: + * + * npx ts-node --esm scripts/codemod-v1-to-v2.ts src/data/iea/IEA-NZE-2024.json ... + * npx ts-node --esm scripts/codemod-v1-to-v2.ts --dry-run src/data/iea + * + * A directory argument expands to the metadata files under it. Files already on v2 + * are skipped, so re-running is safe and the remaining 49 files are a re-run + * rather than a rewrite. + * + * This is a development tool, not a runtime path: the app does not convert v1 + * documents on load. Files that still carry the v1 $schema simply are not loaded + * once the loader points at v2. + * + * Two v1 prose fields are retired without replacement (confirmed with the data + * owner on PR #898): `pathwayOverview`, because it restates the description + * almost verbatim; and the '#### Application to Transition Assessment' section, + * because the new UI does not display it. Neither has a v2 home, so the section + * text is printed in the run report alongside the Core Drivers prose rather than + * being dropped silently. + * + * It also reports, without touching, technologies attached to a sector that does + * not list them (#461). Auto-fixing would mean either deleting a technology + * someone deliberately recorded or inventing a sector's taxonomy -- both worse + * than a line in the report, and `npm run schema:check` refuses the file anyway. + * + * What it does NOT do: populate `coreDrivers`. The v1 "#### Core Drivers" prose + * cannot be mapped onto the 7 named fields mechanically -- several paragraphs + * exceed the 500-char cap, the italic labels do not correspond 1:1 to the field + * names, and each section has unlabeled paragraphs with no destination. Per #858 + * the fields are scaffolded null for hand-authoring; the prose is printed in the + * report so it is not lost track of. + */ +import { promises as fs } from "node:fs"; +import { join, extname } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { PathwayMetadataV1 } from "../src/types/pathwayMetadata.v1.d.ts"; +import type { PathwayMetadataV2 } from "../src/types/pathwayMetadata.v2.d.ts"; +import { technologyBelongsToSector } from "../src/utils/timeseriesTaxonomy.ts"; + +const V1_ID = "http://pathways.rmi.org/schema/pathwayMetadata.v1.json"; +const V2_ID = "http://pathways.rmi.org/schema/pathwayMetadata.v2.json"; + +/** Widest sentinels, mirroring src/utils/validateScopes.ts. */ +const ACROSS_SECTORS = "across sectors"; +const GLOBAL_SCOPE = "Global"; + +/** + * The three sections every v1 `expertOverview` is built from. Verified against all + * 56 files: 55 have all three, and ACE-CNS-2024 is missing "Core Drivers" only + * because its heading lost its `####` markers (see splitExpertOverview). + */ +export const PATHWAY_DESCRIPTION = "Pathway Description"; +export const CORE_DRIVERS = "Core Drivers"; +export const TRANSITION_ASSESSMENT = "Application to Transition Assessment"; +export const SECTION_TITLES = [ + PATHWAY_DESCRIPTION, + CORE_DRIVERS, + TRANSITION_ASSESSMENT, +] as const; + +const CORE_DRIVER_FIELDS = [ + "policies", + "emissionsTargets", + "technologyCosts", + "investmentChange", + "macroeconomicDrivers", + "behavioralShifts", + "otherDrivers", +] as const; + +/** + * Split a v1 `expertOverview` into its named sections. + * + * Headings are ATX (`#### Core Drivers`), but a bare line whose entire content is + * a known section title also counts. That tolerance exists for exactly one file: + * ACE-CNS-2024.json lost the `####` on its "Core Drivers" heading, which would + * otherwise fold 1.5 KB of core-drivers prose into pathwayDescription and push it + * past the 2500-char limit. Treating the bare title as a heading is safe because + * the strings are long and specific enough not to occur as body text. + */ +export function splitExpertOverview(text: string): Map { + const sections = new Map(); + const lines = text.split("\n"); + let current: string | null = null; + let buffer: string[] = []; + + const flush = () => { + if (current !== null) sections.set(current, buffer.join("\n").trim()); + buffer = []; + }; + + for (const line of lines) { + const stripped = line.trim(); + const atx = /^#{1,6}\s*(.+?)\s*$/.exec(stripped); + const heading = atx ? atx[1] : stripped; + const known = SECTION_TITLES.find((t) => t === heading); + if (known && (atx || stripped === known)) { + flush(); + current = known; + continue; + } + if (current !== null) buffer.push(line); + } + flush(); + return sections; +} + +/** + * The widest scope an entry on this pathway can carry, per #858: `across sectors` + * for a multi-sector pathway else its lone sector, and `Global` else the + * pathway's single declared region or country. + * + * Throws rather than guessing when a pathway declares several regions or several + * standalone countries without `global`, since which of them is "widest" is an + * authoring decision (`across regions` exists for that case). No file in the corpus + * hits this today -- all 56 resolve. + */ +export function widestScope(doc: PathwayMetadataV1): { + sector: string; + geography: string; +} { + const names = [...new Set((doc.sectors ?? []).map((s) => s.name))]; + if (names.length === 0) throw new Error("pathway declares no sectors"); + const sector = names.length > 1 ? ACROSS_SECTORS : names[0]; + + const geo = doc.geography ?? {}; + const regions = Object.keys(geo.regions ?? {}); + const countries = geo.country ?? []; + + let geography: string; + if (geo.global === true) { + geography = GLOBAL_SCOPE; + } else if (regions.length === 1) { + geography = regions[0]; + } else if (regions.length > 1) { + throw new Error( + `${regions.length} regions and no "global" flag: widest geography is ambiguous ` + + `(consider "across regions"): ${regions.join(", ")}`, + ); + } else if (countries.length === 1) { + geography = countries[0]; + } else if (countries.length > 1) { + throw new Error( + `${countries.length} countries and no "global" flag or region: widest geography is ambiguous`, + ); + } else { + throw new Error("pathway declares no geography"); + } + + return { sector, geography }; +} + +/** Wrap one v1 value as a single scoped entry at the given scope. */ +function scoped(sector: string, geography: string, value: T) { + return [{ sector, geography, value }]; +} + +export interface UpgradeResult { + doc: PathwayMetadataV2; + /** Prose with no destination in v2 -- reported so it is not lost track of. */ + coreDriversProse: string; + /** Ditto: the new UI does not show this, so v2 has no field for it. */ + transitionAssessmentProse: string; + scope: { sector: string; geography: string }; + /** Chars of `pathwayOverview` discarded, for the run report. */ + droppedPathwayOverview: number; + /** + * `"Sector: Technology"` for each technology its sector does not list (#461). + * Carried straight over from v1, so this reports pre-existing data rather than + * anything the codemod introduced. + */ + unscopedTechnologies: string[]; +} + +/** Transform one v1 document into its v2 equivalent. Pure; does no I/O. */ +export function upgradeV1ToV2(doc: PathwayMetadataV1): UpgradeResult { + const scope = widestScope(doc); + const { sector, geography } = scope; + const kf = doc.keyFeatures; + + const sections = splitExpertOverview(doc.expertOverview ?? ""); + const description = sections.get(PATHWAY_DESCRIPTION) ?? ""; + const assessment = sections.get(TRANSITION_ASSESSMENT) ?? ""; + const coreDriversProse = sections.get(CORE_DRIVERS) ?? ""; + + // `pathwayOverview` is DISCARDED, not folded into pathwayDescription. + // + // #858 says to move it in, and an earlier version of this script did. Reading + // the output showed why that is wrong: the two texts restate each other almost + // verbatim, so the merged field opens by saying the same thing twice. IEA-STEPS + // for instance had "STEPS provides a sense of the energy sector's direction of + // travel today, based on the latest market data, technology costs and in-depth + // analysis of the prevailing policy settings..." immediately followed by "STEPS + // projects the energy sector's current direction of travel, based on the latest + // market data, technology costs and in-depth analysis of the stated policies...". + // + // Confirmed with the data owner (Jacob, PR #898): the field was already out of + // use and should be removed without replacement. It has no readers in the app, + // so nothing observable is lost. + const droppedPathwayOverview = doc.pathwayOverview?.trim().length ?? 0; + + const out: Record = {}; + for (const [key, value] of Object.entries(doc)) { + switch (key) { + case "$schema": + out.$schema = V2_ID; + break; + case "pathwayOverview": + // Superseded; emitted below in expertOverview's position. + break; + case "expertOverview": + out.pathwayDescription = description.length > 0 ? description : null; + break; + case "keyFeatures": + out.keyFeatures = { + emissionsTrajectory: scoped( + sector, + geography, + kf.emissionsTrajectory, + ), + energyEfficiency: scoped(sector, geography, kf.energyEfficiency), + energyDemand: scoped(sector, geography, kf.energyDemand), + electrification: scoped(sector, geography, kf.electrification), + policyTypes: scoped(sector, geography, kf.policyTypes), + technologyCostTrend: scoped( + sector, + geography, + kf.technologyCostTrend, + ), + emissionsScope: scoped(sector, geography, kf.emissionsScope), + policyAmbition: scoped(sector, geography, kf.policyAmbition), + technologyCostsDetail: scoped( + sector, + geography, + kf.technologyCostsDetail, + ), + newTechnologiesIncluded: scoped( + sector, + geography, + kf.newTechnologiesIncluded, + ), + investmentNeeds: scoped(sector, geography, kf.investmentNeeds), + }; + // Scaffolded for hand-authoring; see the file header. + out.coreDrivers = Object.fromEntries( + CORE_DRIVER_FIELDS.map((f) => [f, null]), + ); + out.dependencies = []; + break; + default: + out[key] = value; + } + } + + const unscopedTechnologies = (doc.sectors ?? []).flatMap((s) => + (s.technologies ?? []) + .filter((t) => technologyBelongsToSector(t, s.name) !== "yes") + .map((t) => `${s.name}: ${t}`), + ); + + return { + doc: out as unknown as PathwayMetadataV2, + unscopedTechnologies, + coreDriversProse, + transitionAssessmentProse: assessment, + scope, + droppedPathwayOverview, + }; +} + +async function metadataFilesUnder(path: string): Promise { + const stat = await fs.stat(path); + if (!stat.isDirectory()) return [path]; + const dirents = await fs.readdir(path, { withFileTypes: true }); + const found: string[] = []; + for (const d of dirents) { + const full = join(path, d.name); + if (d.isDirectory()) found.push(...(await metadataFilesUnder(full))); + else if (d.isFile() && extname(d.name) === ".json") found.push(full); + } + return found.sort(); +} + +async function main() { + const args = process.argv.slice(2); + const dryRun = args.includes("--dry-run"); + const paths = args.filter((a) => !a.startsWith("--")); + + if (paths.length === 0) { + console.error( + "usage: codemod-v1-to-v2.ts [--dry-run] [...]\n" + + "Refusing to run with no explicit target.", + ); + process.exit(1); + } + + const files = (await Promise.all(paths.map(metadataFilesUnder))).flat(); + const report: string[] = []; + let migrated = 0; + let skipped = 0; + + for (const file of files) { + const raw = await fs.readFile(file, "utf8"); + const parsed = JSON.parse(raw) as { $schema?: string }; + if (parsed.$schema === V2_ID) { + skipped++; + continue; + } + if (parsed.$schema !== V1_ID) { + // Timeseries files and anything else non-metadata. + skipped++; + continue; + } + + const doc = parsed as unknown as PathwayMetadataV1; + let result: UpgradeResult; + try { + result = upgradeV1ToV2(doc); + } catch (e) { + console.error(`✖ ${file}: ${String(e instanceof Error ? e.message : e)}`); + process.exitCode = 1; + continue; + } + + const { scope, coreDriversProse, droppedPathwayOverview } = result; + const descLength = result.doc.pathwayDescription?.length ?? 0; + console.info( + `✔ ${file}\n` + + ` scope: (${scope.sector}, ${scope.geography})` + + ` pathwayDescription: ${descLength} chars` + + (droppedPathwayOverview > 0 + ? ` [dropped ${droppedPathwayOverview}-char pathwayOverview]` + : ""), + ); + if (result.unscopedTechnologies.length > 0) { + // stderr, because this needs a human before the file will pass + // schema:check -- but not a non-zero exit, since the migration itself + // succeeded and the offending data predates it. + console.error( + ` #461: technologies outside their sector's list, left as-is: ` + + result.unscopedTechnologies.join("; "), + ); + } + + const orphaned: string[] = []; + if (coreDriversProse.length > 0) { + orphaned.push(`### ${CORE_DRIVERS}\n\n${coreDriversProse}`); + } else { + console.info(` note: no "${CORE_DRIVERS}" section found`); + } + if (result.transitionAssessmentProse.length > 0) { + orphaned.push( + `### ${TRANSITION_ASSESSMENT}\n\n${result.transitionAssessmentProse}`, + ); + } + if (orphaned.length > 0) { + report.push(`## ${file}\n\n${orphaned.join("\n\n")}\n`); + } + + if (!dryRun) { + await fs.writeFile(file, `${JSON.stringify(result.doc, null, 2)}\n`); + } + migrated++; + } + + console.info( + `\n${dryRun ? "Would migrate" : "Migrated"} ${migrated} file(s); skipped ${skipped}.`, + ); + + if (report.length > 0) { + console.info( + "\n--- v1 prose with no v2 destination ---\n" + + `"${CORE_DRIVERS}" becomes the structured coreDrivers object, scaffolded null here;\n` + + `"${TRANSITION_ASSESSMENT}" is retired because the new UI does not display it.\n` + + "Both are reproduced below so the text is not lost.\n", + ); + console.info(report.join("\n")); + } + if (!dryRun) { + console.info("Run `npm run schema:check` and `npx prettier --write` next."); + } +} + +// Only run when invoked directly, so the exported helpers stay importable by tests. +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main().catch((e: unknown) => { + console.error(String(e instanceof Error ? e.stack : e)); + process.exit(1); + }); +} diff --git a/scripts/import-benchmark-data.test.ts b/scripts/import-benchmark-data.test.ts new file mode 100644 index 00000000..2d86d7f9 --- /dev/null +++ b/scripts/import-benchmark-data.test.ts @@ -0,0 +1,298 @@ +import { describe, it, expect } from "vitest"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + commitWrites, + planImport, + type Corpus, + type WriteFs, +} from "./import-benchmark-data.ts"; +import { Report } from "./import-pathway-data.ts"; +import type { PathwayMetadataV2 } from "../src/types/pathwayMetadata.v2.d.ts"; +// A real publication block: publishers are a closed list in the schema. +import ieaSteps from "../src/data/iea/IEA-STEPS-2024_timeseries.json" with { type: "json" }; + +const V2 = "http://pathways.rmi.org/schema/pathwayTimeseries.v2.json"; + +const row = (over: Record = {}) => ({ + year: 2030, + geography: "Southeast Asia", + sector: "power", + sectorSegment: ["Power generation"], + technology: null, + metric: "capacity", + value: 1, + unit: "GW", + ...over, +}); + +const file = (over: Record = {}, rows = [row()]) => ({ + $schema: V2, + id: "IEA-X_timeseries", + pathwayId: ["IEA-X"], + name: "IEA X Timeseries Data", + description: "A test series.", + publication: ieaSteps.publication, + pathwayName: "X", + emissionsScope: "CO2", + data: rows, + ...over, +}); + +const corpus = (existing = false): Corpus => ({ + metadataGeographyById: new Map([ + ["IEA-X", { global: true, regions: { "Southeast Asia": ["TH", "VN"] } }], + ]), + metadataPathById: new Map([["IEA-X", "src/data/iea/IEA-X.json"]]), + occupiedPaths: new Set([ + "src/data/iea/IEA-X.json", + ...(existing + ? [ + "src/data/iea/X-legacy-name_timeseries.json", + "src/data/iea/OTHER.json", + ] + : []), + ]), + timeseriesById: new Map( + existing + ? [ + [ + "IEA-X_timeseries", + { path: "src/data/iea/X-legacy-name_timeseries.json", doc: {} }, + ], + ["OTHER_timeseries", { path: "src/data/iea/OTHER.json", doc: {} }], + ] + : [], + ), +}); + +const plan = (inputs: unknown[], c = corpus()) => { + const report = new Report(); + const writes = planImport( + inputs.map((data, i) => ({ name: `in/${i}.json`, data })), + c, + report, + ); + return { writes, report }; +}; + +describe("planImport", () => { + it("places a new file next to its pathway's metadata", () => { + const { writes, report } = plan([file()]); + expect(report.errors).toEqual([]); + expect(writes.map((w) => [w.path, w.replaces])).toEqual([ + ["src/data/iea/IEA-X_timeseries.json", false], + ]); + }); + + it("overwrites an existing file in place, keeping its name", () => { + const { writes, report } = plan([file()], corpus(true)); + expect(writes.map((w) => w.path)).toEqual([ + "src/data/iea/X-legacy-name_timeseries.json", + ]); + // ...and says which existing files the import leaves alone. + expect(report.lines.join("\n")).toContain("left as they are: OTHER.json"); + }); + + it("never lets a new id land on an existing file with another id", () => { + // ACE's real shape: the file name drops the id's prefix, so an input whose + // id happens to equal that name is new by id but not by path. + const { writes, report } = plan( + [file({ id: "X-legacy-name_timeseries" })], + corpus(true), + ); + expect(report.errors.join("\n")).toMatch( + /X-legacy-name_timeseries\.json is already taken/, + ); + expect(writes).toEqual([]); + }); + + it("treats a path differing only by case as taken", () => { + // On default macOS and Windows file systems these are one file. + const { writes, report } = plan([file({ id: "iea-x" })]); + expect(report.errors.join("\n")).toMatch( + /src\/data\/iea\/iea-x\.json is already taken/, + ); + expect(writes).toEqual([]); + }); + + it("reports a taken destination in the same run as the file's other errors", () => { + const { report } = plan( + [ + file({ id: "X-legacy-name_timeseries" }, [ + row({ geography: "Atlantis" }), + ]), + ], + corpus(true), + ); + const errors = report.errors.join("\n"); + expect(errors).toMatch(/"Atlantis" is not a geography/); + expect(errors).toMatch(/is already taken/); + }); + + it.each([ + [ + "an undeclared geography", + [row({ geography: "South East Asia" })], + /not a geography pathway IEA-X declares/, + ], + [ + "a segment of another sector", + [row({ sectorSegment: ["Ironmaking"] })], + /not a segment of Power/, + ], + [ + "a sentinel segment", + [row({ sectorSegment: ["Unspecified"] })], + /is not allowed/, + ], + [ + "a v1-shaped row with no sectorSegment", + [{ ...row(), sectorSegment: undefined }], + /sectorSegment/, + ], + ])("blocks the import on %s", (_, rows, message) => { + const { writes, report } = plan([file({}, rows)]); + expect(report.errors.join("\n")).toMatch(message); + expect(writes).toEqual([]); + }); + + it("blocks a file whose pathway has no metadata", () => { + const { report } = plan([file({ pathwayId: ["GONE"] })]); + expect(report.errors.join("\n")).toMatch( + /"GONE" is not the id of any pathway/, + ); + }); + + it("blocks a file still on the v1 schema", () => { + const { report } = plan([ + file({ + $schema: "http://pathways.rmi.org/schema/pathwayTimeseries.v1.json", + }), + ]); + expect(report.errors.length).toBeGreaterThan(0); + }); + + it("blocks two input files with the same id, and still checks the second", () => { + // One run should list every problem, so the duplicate's own errors are + // reported alongside the duplicate id. + const { writes, report } = plan([ + file(), + file({}, [row({ geography: "Atlantis" })]), + ]); + const errors = report.errors.join("\n"); + expect(errors).toMatch(/also used by in\/0\.json/); + expect(errors).toMatch(/"Atlantis" is not a geography/); + expect(writes.map((w) => w.path)).toEqual([ + "src/data/iea/IEA-X_timeseries.json", + ]); + }); + + it.each(["../escape", "iea/nested", ".hidden"])( + "blocks an id that is not a safe file name: %s", + (id) => { + const { writes, report } = plan([file({ id })]); + expect(report.errors.join("\n")).toMatch(/cannot be used as a file name/); + expect(writes).toEqual([]); + }, + ); +}); + +describe("commitWrites", () => { + /** An in-memory file system whose writes fail for chosen paths. */ + const memoryFs = (files: Record, failOn: string[] = []) => { + const failing = new Set(failOn); // each fails once; the restore succeeds + const io: WriteFs = { + readFile: async (path) => files[path] ?? null, + writeFile: async (path, text) => { + if (failing.delete(path)) { + // Like ENOSPC: the file is truncated before the write gives up. + files[path] = text.slice(0, 3); + throw new Error(`disk full at ${path}`); + } + files[path] = text; + }, + removeFile: async (path) => { + delete files[path]; + }, + }; + return { files, io }; + }; + + it("writes every staged file", async () => { + const { files, io } = memoryFs({ a: "old a" }); + await commitWrites( + [ + { path: "a", text: "new a" }, + { path: "b", text: "new b" }, + ], + io, + ); + expect(files).toEqual({ a: "new a", b: "new b" }); + }); + + it("restores earlier files and removes new ones when a later write fails", async () => { + const { files, io } = memoryFs({ a: "old a", c: "old c" }, ["c"]); + await expect( + commitWrites( + [ + { path: "a", text: "new a" }, + { path: "b", text: "new b" }, + { path: "c", text: "new c" }, + ], + io, + ), + ).rejects.toThrow(/the 3 file\(s\) touched were restored/); + // c included: its failed write had already truncated it. + expect(files).toEqual({ a: "old a", c: "old c" }); + }); + + it("restores the file whose own write failed, even when it is the first", async () => { + const { files, io } = memoryFs({ a: "old a" }, ["a"]); + await expect( + commitWrites([{ path: "a", text: "new a" }], io), + ).rejects.toThrow(); + expect(files).toEqual({ a: "old a" }); + }); + + it("writes nothing when an existing file cannot be backed up", async () => { + const { files, io } = memoryFs({ a: "old a" }); + io.readFile = async (path) => { + if (path === "b") throw new Error("permission denied"); + return files[path] ?? null; + }; + await expect( + commitWrites( + [ + { path: "a", text: "new a" }, + { path: "b", text: "new b" }, + ], + io, + ), + ).rejects.toThrow(/permission denied/); + expect(files).toEqual({ a: "old a" }); + }); + + it("does not delete an existing file it could not back up", async () => { + // A write-only file exists but cannot be read (EACCES). Counting that as + // "new" would make a rollback delete it; it must abort before writing. + const dir = await mkdtemp(join(tmpdir(), "import-benchmark-")); + const locked = join(dir, "locked.json"); + try { + await writeFile(locked, "old"); + await chmod(locked, 0o222); + await expect( + commitWrites([ + { path: locked, text: "new" }, + // Missing on read (so "new"), and its write fails: no such folder. + { path: join(dir, "no-such-folder", "x.json"), text: "{}" }, + ]), + ).rejects.toThrow(/EACCES/); + await chmod(locked, 0o644); + expect(await readFile(locked, "utf8")).toBe("old"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/import-benchmark-data.ts b/scripts/import-benchmark-data.ts new file mode 100644 index 00000000..5566b0e4 --- /dev/null +++ b/scripts/import-benchmark-data.ts @@ -0,0 +1,367 @@ +/** + * Import benchmark timeseries prepared by RMI/tpr_benchmark_data_preparation + * into src/data (#915, #902). + * + * Usage: + * npx ts-node --esm scripts/import-benchmark-data.ts --in [--dry-run] + * + * `` holds the prep repo's output: one JSON file per timeseries dataset, + * already in `pathwayTimeseries.v2.json` format. This script does not convert + * anything. It is the gate between the two repos: every file must pass the v2 + * schema and the cross-file checks in src/utils/validateTimeseries.ts (geography + * declared by the pathway's metadata, segments belonging to the row's sector) + * against the metadata already in src/data. + * + * Where each file goes: + * - an existing src/data timeseries with the same `id` is overwritten in + * place, keeping its file name (some, like ACE's ATS-2024_timeseries.json, + * do not follow the id); + * - otherwise a new `.json` is written next to the metadata of the + * file's first pathway (ids already end in `_timeseries`). + * Existing timeseries absent from the input are reported, never deleted. + * + * Like import-pathway-data.ts, any error blocks the whole import: nothing is + * written, every error is listed, and the run exits 1. A half-imported data + * set is worse than none. With --dry-run it reports and writes nothing either + * way, which is how the prep repo can check its output against TPR's contract. + */ +import { promises as fs } from "node:fs"; +import { basename, dirname, join, normalize } from "node:path"; +import { fileURLToPath } from "node:url"; +import * as prettier from "prettier"; +import { Report } from "./import-pathway-data.ts"; +import { validateFilesBySchema } from "../src/utils/validateData.ts"; +import { + PATHWAY_TIMESERIES_V2_ID, + validateTimeseries, + type MetadataGeographyById, +} from "../src/utils/validateTimeseries.ts"; +import { PATHWAY_METADATA_V2_ID } from "../src/utils/validateScopes.ts"; +import pathwayMetadataV1Schema from "../src/schema/pathwayMetadata.v1.json" with { type: "json" }; +import pathwayMetadataV2Schema from "../src/schema/pathwayMetadata.v2.json" with { type: "json" }; +import pathwayTimeseriesV2Schema from "../src/schema/pathwayTimeseries.v2.json" with { type: "json" }; +import { commonSchemas } from "../src/schema/common/index.ts"; +import type { PathwayMetadataV2 } from "../src/types/pathwayMetadata.v2.d.ts"; +import type { PathwayTimeseriesV2 } from "../src/types/pathwayTimeseries.v2.d.ts"; + +const DATA_DIR = "src/data"; +const METADATA_IDS = new Set([ + String(pathwayMetadataV1Schema.$id), + PATHWAY_METADATA_V2_ID, +]); +const TIMESERIES_ID_PATTERN = /pathwayTimeseries\.v\d+\.json$/; +const SAFE_FILE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +type Json = Record; + +/** What src/data already holds, as far as an import needs to know. */ +export interface Corpus { + /** Each pathway's declared geography, by pathway id. */ + metadataGeographyById: MetadataGeographyById; + /** Where each pathway's metadata file lives, by pathway id. */ + metadataPathById: ReadonlyMap; + /** Existing timeseries files, by dataset `id`. */ + timeseriesById: ReadonlyMap; + /** + * Every file path under src/data. A new file must not land on one: file + * names do not always follow ids (ACE's `ATS-2024_timeseries.json` holds + * `ACE-ATS-2024_timeseries`), so a free path cannot be inferred from ids. + */ + occupiedPaths: ReadonlySet; +} + +export interface InputFile { + name: string; + data: unknown; +} + +export interface PlannedWrite { + path: string; + doc: PathwayTimeseriesV2; + replaces: boolean; +} + +const distinct = (values: Iterable) => [...new Set(values)].sort(); + +/** A path as a case-insensitive file system sees it. */ +const pathKey = (path: string) => normalize(path).toLowerCase(); + +function describe(doc: { data?: unknown }): string { + const rows = Array.isArray(doc.data) ? (doc.data as Json[]) : []; + const geographies = distinct(rows.map((r) => String(r.geography))); + const segments = distinct( + rows.flatMap((r) => + Array.isArray(r.sectorSegment) ? (r.sectorSegment as string[]) : [], + ), + ); + return ( + `${rows.length} rows; geography ${geographies.join(", ") || "-"}` + + (segments.length ? `; segments ${segments.join(", ")}` : "") + ); +} + +/** + * Validate the input files and decide where each goes. Pure apart from the + * report: errors are recorded there, and the caller writes only when there + * are none. + */ +export function planImport( + inputs: readonly InputFile[], + corpus: Corpus, + report: Report, +): PlannedWrite[] { + const { valid, invalid } = validateFilesBySchema( + [...inputs], + [ + pathwayMetadataV1Schema, + pathwayMetadataV2Schema, + pathwayTimeseriesV2Schema, + ...commonSchemas, + ], + ); + for (const problem of invalid) + for (const e of problem.errors) report.error(`${problem.name}: ${e}`); + + const planned: PlannedWrite[] = []; + // Paths compared ignoring case: on the default macOS and Windows file + // systems `iea-x.json` and `IEA-X.json` are one file. + const occupiedKeys = new Set([...corpus.occupiedPaths].map(pathKey)); + const plannedKeys = new Set(); + const seenIds = new Map(); + for (const record of valid) { + if (record.schemaId !== PATHWAY_TIMESERIES_V2_ID) { + report.error( + `${record.name}: $schema is ${record.schemaId}, expected ${PATHWAY_TIMESERIES_V2_ID}`, + ); + continue; + } + const doc = record.data as PathwayTimeseriesV2; + // Every check runs before the file is skipped, so one run lists every + // problem rather than the first one found. + const problems = validateTimeseries(doc, corpus.metadataGeographyById); + const earlier = seenIds.get(doc.id); + if (earlier) problems.push(`id "${doc.id}" is also used by ${earlier}`); + else seenIds.set(doc.id, record.name); + // A new file is named after its id, so the id must not be able to point + // anywhere else (no "/" or a leading "."). + if (!SAFE_FILE_ID.test(doc.id)) + problems.push( + `id "${doc.id}" cannot be used as a file name: use letters, digits, ".", "_" and "-", starting with a letter or digit`, + ); + // Where the file would go, worked out even when the checks above failed, + // so a placement problem is reported in the same run rather than the next. + const existing = corpus.timeseriesById.get(doc.id); + let path = existing?.path; + if (!existing) { + const metadataPath = corpus.metadataPathById.get(doc.pathwayId[0]); + if (metadataPath === undefined) { + // validateTimeseries reports unknown pathway ids; this only guards + // against that ever changing. + if (problems.length === 0) + problems.push(`no metadata file for ${doc.pathwayId[0]}`); + } else if (SAFE_FILE_ID.test(doc.id)) { + path = join(dirname(metadataPath), `${doc.id}.json`); + const key = pathKey(path); + if (occupiedKeys.has(key) || plannedKeys.has(key)) + problems.push( + `id "${doc.id}" is new, but ${path} is already taken by another file ` + + "(names compared ignoring case); an update must use the existing file's id", + ); + plannedKeys.add(key); + } + } + + for (const e of problems) report.error(`${record.name}: ${e}`); + if (problems.length > 0 || path === undefined) continue; + planned.push({ path, doc, replaces: existing !== undefined }); + report.note( + existing + ? `UPD ${path}: ${describe(existing.doc)} -> ${describe(doc)}` + : `NEW ${path}: ${describe(doc)}`, + ); + } + + const imported = new Set(seenIds.keys()); + const untouched = [...corpus.timeseriesById] + .filter(([id]) => !imported.has(id)) + .map(([, { path }]) => basename(path)); + if (untouched.length) + report.note( + `Not in this import, left as they are: ${untouched.sort().join(", ")}`, + ); + return planned; +} + +/** The file operations {@link commitWrites} needs; injectable for tests. */ +export interface WriteFs { + readFile(path: string): Promise; + writeFile(path: string, text: string): Promise; + removeFile(path: string): Promise; +} + +const nodeFs: WriteFs = { + readFile: async (path) => { + try { + return await fs.readFile(path, "utf8"); + } catch (error) { + // Only a missing file is "new". Any other failure (an unreadable but + // writable file, say) must stop the import: without a backup, a + // rollback would delete the file instead of restoring it. + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + }, + writeFile: (path, text) => fs.writeFile(path, text), + removeFile: (path) => fs.rm(path, { force: true }), +}; + +/** + * Write every staged file, or none: if any write fails, every file touched so + * far — including the one whose write failed, which may already be truncated + * — is restored to what it held before (or removed, if new), and the error is + * rethrown. This is what makes the import's all-or-nothing promise hold past + * validation, through to the disk. A restore that itself fails does not stop + * the others; the error names the files left as they are. + */ +export async function commitWrites( + staged: readonly { path: string; text: string }[], + io: WriteFs = nodeFs, +): Promise { + // Read every backup before writing anything; a read failure aborts here, + // with nothing touched yet. + const originals = await Promise.all( + staged.map(async ({ path }) => ({ path, text: await io.readFile(path) })), + ); + const touched: number[] = []; + try { + for (const [i, { path, text }] of staged.entries()) { + touched.push(i); // before the write: a failed write may still have changed the file + await io.writeFile(path, text); + } + } catch (error) { + const unrestored: string[] = []; + for (const i of touched.reverse()) { + const { path, text } = originals[i]; + try { + if (text === null) await io.removeFile(path); + else await io.writeFile(path, text); + } catch { + unrestored.push(path); + } + } + const outcome = unrestored.length + ? `could NOT restore ${unrestored.join(", ")}; check them with git` + : `the ${touched.length} file(s) touched were restored`; + throw new Error(`writing failed, and ${outcome}: ${String(error)}`); + } +} + +async function jsonFilesUnder(dir: string): Promise { + const out: string[] = []; + for (const d of await fs.readdir(dir, { withFileTypes: true })) { + const full = join(dir, d.name); + if (d.isDirectory()) out.push(...(await jsonFilesUnder(full))); + else if (d.name.endsWith(".json")) out.push(full); + } + return out; +} + +/** Read src/data into the shape {@link planImport} needs. */ +export async function readCorpus(dir = DATA_DIR): Promise { + const metadataGeographyById = new Map< + string, + PathwayMetadataV2["geography"] + >(); + const metadataPathById = new Map(); + const timeseriesById = new Map(); + const occupiedPaths = new Set(); + for (const path of await jsonFilesUnder(dir)) { + occupiedPaths.add(path); + let doc: Json; + try { + doc = JSON.parse(await fs.readFile(path, "utf8")) as Json; + } catch { + continue; // not a data document; schema:check reports it + } + const schema = String(doc.$schema ?? ""); + if (METADATA_IDS.has(schema)) { + metadataGeographyById.set( + String(doc.id), + doc.geography as PathwayMetadataV2["geography"], + ); + metadataPathById.set(String(doc.id), path); + } else if (TIMESERIES_ID_PATTERN.test(schema)) { + timeseriesById.set(String(doc.id), { path, doc }); + } + } + return { + metadataGeographyById, + metadataPathById, + timeseriesById, + occupiedPaths, + }; +} + +async function main() { + const args = process.argv.slice(2); + const dryRun = args.includes("--dry-run"); + const inAt = args.indexOf("--in"); + const inDir = inAt >= 0 ? args[inAt + 1] : undefined; + if (!inDir) { + console.error("Usage: import-benchmark-data.ts --in [--dry-run]"); + process.exit(2); + } + + const report = new Report(); + const inputs: InputFile[] = []; + for (const path of await jsonFilesUnder(inDir)) { + try { + inputs.push({ + name: path, + data: JSON.parse(await fs.readFile(path, "utf8")), + }); + } catch (e) { + report.error(`${path}: not valid JSON (${String(e)})`); + } + } + report.section(`Import of ${inputs.length} file(s) from ${inDir}`); + const planned = planImport(inputs, await readCorpus(), report); + + console.info(report.lines.join("\n")); + if (report.errors.length > 0) { + console.error( + `\n## ${report.errors.length} error(s) -- nothing written\n` + + "Fix these in the prepared data and re-run the import:\n" + + report.errors.map((e) => ` - ${e}`).join("\n"), + ); + process.exitCode = 1; + return; + } + if (dryRun) { + console.info(`\nDry run: ${planned.length} file(s) would be written.`); + return; + } + // Format everything before touching any file, so a formatting failure + // cannot leave a half-written import behind. + const staged = await Promise.all( + planned.map(async ({ path, doc }) => { + const options = (await prettier.resolveConfig(path)) ?? {}; + const text = await prettier.format(JSON.stringify(doc, null, 2), { + ...options, + parser: "json", + }); + return { path, text }; + }), + ); + await commitWrites(staged); + console.info( + `\nWrote ${planned.length} file(s). Run \`npm run build:timeseries\` and \`npm run schema:check\` next.`, + ); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main().catch((e: unknown) => { + console.error(String(e instanceof Error ? e.stack : e)); + process.exit(1); + }); +} diff --git a/scripts/import-pathway-data.test.ts b/scripts/import-pathway-data.test.ts new file mode 100644 index 00000000..96cfae48 --- /dev/null +++ b/scripts/import-pathway-data.test.ts @@ -0,0 +1,372 @@ +import { describe, it, expect } from "vitest"; +import { + Report, + publisherGroup, + scenarioParts, + canonicalKey, + matchEnum, + normalizeSector, + splitList, + ensurePeriod, + isAbsent, + resolveGeography, + buildGeography, + TARGETS, + parseSegments, + parseCoverage, + parseGranularity, + buildDataAvailability, + descriptionFromV1, + keyFeaturesFromV1, +} from "./import-pathway-data.ts"; + +describe("publisherGroup", () => { + it("maps each sheet's spelling to one group", () => { + expect(publisherGroup("International Energy Agency (IEA)")).toBe("IEA"); + expect(publisherGroup("ASEAN Centre for Energy (ACE)")).toBe("ACE"); + expect(publisherGroup("ACE")).toBe("ACE"); + // AGF must win over the SDSN check even though both live under un-sdsn-cw. + expect(publisherGroup("ASEAN Green Future (AGF)")).toBe("AGF"); + expect(publisherGroup("UN SDSN, CW")).toBe("SDSN"); + expect(publisherGroup("Some Unknown Org")).toBeNull(); + }); +}); + +describe("scenarioParts", () => { + it("reads the trailing (CODE) and [CC]", () => { + expect(scenarioParts("Current Policies Scenario (CPS)")).toEqual({ + code: "CPS", + country: "", + }); + expect( + scenarioParts("Optimised More Ambitious Pathway (OMAP) [MM]"), + ).toEqual({ code: "OMAP", country: "MM" }); + }); + it("derives JRC codes from names with no parenthetical", () => { + expect(scenarioParts("Reference scenario").code).toBe("REFERENCE"); + expect(scenarioParts("NDC-LTS scenario").code).toBe("NDC-LTS"); + expect(scenarioParts("1.5°C scenario").code).toBe("1.5C"); + }); +}); + +describe("canonicalKey", () => { + it("joins the sheets across their name/publisher variations", () => { + // "State" vs "States" typo in the sub-sheets must not break the join. + expect( + canonicalKey("ACE", "ASEAN Member State Targets Scenario (ATS)"), + ).toBe("ACE:ATS:"); + expect( + canonicalKey( + "ASEAN Centre for Energy (ACE)", + "ASEAN Member States Targets Scenario (ATS)", + ), + ).toBe("ACE:ATS:"); + // Myanmar and Laos OMAP resolve to distinct keys. + expect( + canonicalKey( + "ASEAN Green Future (AGF)", + "Optimised More Ambitious Pathway (OMAP) [MM]", + ), + ).toBe("AGF:OMAP:MM"); + expect( + canonicalKey( + "ASEAN Green Future (AGF)", + "Optimized More Ambitious Policy (OMAP)", + ), + ).toBe("AGF:OMAP:"); + }); +}); + +describe("value normalization", () => { + it("matches enums case-insensitively and fixes the 'Signifcant' typo", () => { + const demand = ["Significant decrease", "Low or no change"]; + expect(matchEnum("Signifcant Decrease", demand)).toBe( + "Significant decrease", + ); + expect(matchEnum("carbon price", ["Carbon price"])).toBe("Carbon price"); + expect(matchEnum("nonsense", demand)).toBeNull(); + }); + it("treats only NULL / n/a / blank as absent", () => { + expect(isAbsent("NULL")).toBe(true); + expect(isAbsent("n/a")).toBe(true); + expect(isAbsent("")).toBe(true); + expect(isAbsent(" ")).toBe(true); + expect(isAbsent("No information")).toBe(false); + }); + + it("does not treat the scope sentinels as absent", () => { + // Cookbook decision 0023 made these authored values, and #858 added them to + // the schema. They used to be swallowed here, which discarded the very + // distinction they exist to draw: an explicit value terminates #869's + // fallback chain where an absent entry keeps broadening. + expect(isAbsent("Not applicable at this scope level")).toBe(false); + expect(isAbsent("Not Applicable")).toBe(false); + }); + + it("reports a pre-0023 spelling rather than hiding it", () => { + // Not absent, and not an enum member either, so it lands in the importer's + // badValues report -- which is what names the cells still to correct in the + // workbook. Previously it disappeared silently. + for (const retired of [ + "Not available at this scope", + "Not applicable at that scope level", + ]) { + expect(isAbsent(retired)).toBe(false); + expect(matchEnum(retired, ["Not applicable at this scope level"])).toBe( + null, + ); + } + }); + it("ensures a trailing period and splits mixed-delimiter lists", () => { + expect(ensurePeriod("foo")).toBe("foo."); + expect(ensurePeriod("foo.")).toBe("foo."); + expect(splitList("Carbon price; Subsidies, performance standards")).toEqual( + ["Carbon price", "Subsidies", "performance standards"], + ); + expect(normalizeSector("power")).toBe("Power"); + expect(normalizeSector("nope")).toBeNull(); + }); +}); + +describe("buildGeography", () => { + it("parses the Regions cell, dropping non-ISO codes", () => { + const report = new Report(); + const geo = buildGeography( + "Global; Africa: [DZ, EG, XK]; PH", + report, + "TEST", + ); + expect(geo?.global).toBe(true); + expect(geo?.regions?.Africa).toEqual(["DZ", "EG"]); // XK dropped + expect(geo?.country).toEqual(["PH"]); + expect(report.lines.some((l) => l.includes("XK"))).toBe(true); + }); + it("resolves geography tokens against declared coverage, ignoring punctuation", () => { + const allowed = new Set(["Rest Sub-Saharan Africa", "TH", "Global"]); + expect(resolveGeography("Rest Sub Saharan Africa", allowed)).toBe( + "Rest Sub-Saharan Africa", + ); + expect(resolveGeography("th", allowed)).toBe("TH"); + expect(resolveGeography("EFTA", allowed)).toBeNull(); + }); + it("reads `LA: [LA]` as the country it is, not a region named after it", () => { + // The cookbook's mandated notation for an individually projected country. + // Read as a region, every single-country pathway became a pseudo-region -- + // except Thailand, whose cell is written bare. + const geo = buildGeography("LA: [LA]", new Report(), "TEST"); + expect(geo?.country).toEqual(["LA"]); + expect(geo?.regions).toBeUndefined(); + // A real region, and a region whose label is a code but has several + // members, stay regions. + const multi = buildGeography( + "ASEAN: [TH, VN]; EU: [FR, DE]", + new Report(), + "T", + ); + expect(Object.keys(multi?.regions ?? {})).toEqual(["ASEAN", "EU"]); + expect(multi?.country).toBeUndefined(); + }); +}); + +describe("TARGETS", () => { + it("matches every row of the current workbook's renamed SDSN spellings", () => { + // The workbook now writes the Myanmar/Laos pathways as "UN SDSN, CW" with + // the country inside the code. Under the old "AGF:...:MM" keys all seven + // silently stopped matching, and the six Singapore files were never listed. + for (const name of [ + "Existing Policies Pathway (EPP MM)", + "Optimised More Ambitious Pathway (OMAP MM)", + "Optimized More Ambitious Policy Scenario (OMAP LA)", + "Existing Policy Scenario (EP LA)", + "Baseline Scenario (BAS SG)", + "Highly Ambitious Scenario 2 Simulated (HA2S SG)", + ]) { + const key = canonicalKey("UN SDSN, CW", name); + expect(key && TARGETS[key], `${name} -> ${key}`).toBeTruthy(); + } + }); +}); + +describe("descriptionFromV1", () => { + const overview = + "#### Pathway Description\n\nA pathway about Laos\n\n#### Core Drivers\n\nPrices."; + it("takes the Pathway Description section of a v1 expertOverview", () => { + const report = new Report(); + expect(descriptionFromV1({ expertOverview: overview }, report, "T")).toBe( + "A pathway about Laos.", + ); + expect(report.lines.some((l) => l.includes("taken from v1"))).toBe(true); + }); + it("returns null when there is no v1 text to fall back on", () => { + expect(descriptionFromV1({}, new Report(), "T")).toBeNull(); + expect(descriptionFromV1(null, new Report(), "T")).toBeNull(); + }); +}); + +describe("keyFeaturesFromV1", () => { + const v1 = { + $schema: "http://pathways.rmi.org/schema/pathwayMetadata.v1.json", + sectors: [{ name: "Power" }], + geography: { country: ["PH"] }, + keyFeatures: { + emissionsTrajectory: "Moderate increase", + policyTypes: ["Carbon price", "Subsidies"], + }, + }; + it("keeps a v1 file's authored values as widest-scope entries", () => { + const report = new Report(); + const kf = keyFeaturesFromV1(v1, report, "T"); + expect(kf?.emissionsTrajectory).toEqual([ + { sector: "Power", geography: "PH", value: "Moderate increase" }, + ]); + expect(kf?.policyTypes).toEqual([ + { + sector: "Power", + geography: "PH", + value: ["Carbon price", "Subsidies"], + }, + ]); + expect(report.lines.some((l) => l.includes("converted from v1"))).toBe( + true, + ); + }); + it("leaves a v2 file to the workbook", () => { + const v2 = { + ...v1, + $schema: "http://pathways.rmi.org/schema/pathwayMetadata.v2.json", + }; + expect(keyFeaturesFromV1(v2, new Report(), "T")).toBeNull(); + expect(keyFeaturesFromV1(null, new Report(), "T")).toBeNull(); + }); +}); + +describe("data-availability cell parsing", () => { + it("unwraps `Sector: [..]` and splits segments on ; or ,", () => { + const bad = new Set(); + expect( + parseSegments("Power: [Power generation, Energy storage]", "Power", bad), + ).toEqual(["Power generation", "Energy storage"]); + expect( + parseSegments("power generation; Energy Storage", "Power", bad), + ).toEqual(["Power generation", "Energy storage"]); + expect(bad.size).toBe(0); + }); + it("takes the label of `Label: [members]` coverage and normalises sentinel case", () => { + const allowed = new Set(["SG", "ASEAN"]); + const bad = new Set(); + expect(parseCoverage("SG: [SG]", allowed, bad)).toEqual(["SG"]); + expect(parseCoverage("Not Covered", allowed, bad)).toEqual(["Not covered"]); + expect(parseCoverage("ASEAN; Atlantis", allowed, bad)).toEqual(["ASEAN"]); + expect([...bad]).toEqual(['geography:"Atlantis"']); + }); + it("does not split granularity values that contain commas", () => { + const bad = new Set(); + expect(parseGranularity("Capital costs, O&M, etc.", bad)).toEqual([ + "Capital costs, O&M, etc.", + ]); + expect(parseGranularity("Coal / Oil / Gas", bad)).toEqual([ + "Coal", + "Oil", + "Gas", + ]); + // A comma list is split only when every part is a known value. + expect(parseGranularity("Coal, Solar", bad)).toEqual(["Coal", "Solar"]); + expect(bad.size).toBe(0); + parseGranularity("Coal, Moonbeams", bad); + expect([...bad]).toEqual(['granularity:"Coal, Moonbeams"']); + }); +}); + +describe("buildDataAvailability", () => { + const row = (over: Record = {}) => ({ + "Sector": "Power", + "Metric": "Capacity", + "Sector segment": "Power generation", + "Granularity": "Coal; Solar", + "Scope limitations": "Grid-connected only.", + "Geography coverage": "TH", + "Time resolution": "5-year steps", + "Data Format": "Tabular", + ...over, + }); + const build = (rows: Record[], sectors = ["Power"]) => { + const report = new Report(); + const out = buildDataAvailability( + rows, + new Set(sectors), + new Set(["TH"]), + report, + "T", + ); + return { + out, + notes: report.lines.join("\n"), + errors: report.errors.join("\n"), + }; + }; + + it("emits a complete row with schema-cased values", () => { + const { out } = build([row({ Metric: "absolute emissions" })]); + expect(out?.overall).toBeNull(); + expect(out?.byMetric).toEqual([ + { + metricName: "Absolute Emissions", + sector: "Power", + sectorSegment: ["Power generation"], + geography: ["TH"], + timeResolution: "5-year steps", + dataFormat: "Tabular", + granularity: ["Coal", "Solar"], + scopeLimitations: "Grid-connected only.", + }, + ]); + }); + it("matches the metric against the row's own sector first", () => { + // Power spells it "Absolute Emissions", Steel "Absolute emissions"; a flat + // case-insensitive match would hand Steel the Power spelling. + const { out } = build( + [ + row({ + Sector: "Steel", + Metric: "Absolute Emissions", + Granularity: "Scope 1", + }), + ], + ["Steel"], + ); + expect(out?.byMetric[0].metricName).toBe("Absolute emissions"); + }); + it("reports a row with no Data Format as an error rather than inventing one", () => { + const { out, errors } = build([ + row(), + row({ "Metric": "Generation", "Data Format": "" }), + ]); + expect(out?.byMetric.map((r) => r.metricName)).toEqual(["Capacity"]); + expect(errors).toContain( + "T/dataAvailability Power Generation: blank or unrecognised data format", + ); + }); + it("reports a row that is Not covered on some variables only as an error", () => { + const { out, errors } = build([row({ "Data Format": "Not covered" })]); + expect(out).toBeNull(); + expect(errors).toContain('"Not covered" on data format only'); + }); + it("keeps a row that is Not covered on every variable", () => { + const nc = "Not covered"; + const { out, errors } = build([ + row({ + "Sector segment": nc, + "Granularity": nc, + "Scope limitations": nc, + "Geography coverage": "Not Covered", + "Time resolution": nc, + "Data Format": nc, + }), + ]); + expect(out?.byMetric).toHaveLength(1); + expect(errors).toBe(""); + }); + it("returns null when the sheet has no rows for the pathway", () => { + expect(build([]).out).toBeNull(); + }); +}); diff --git a/scripts/import-pathway-data.ts b/scripts/import-pathway-data.ts new file mode 100644 index 00000000..39b74fce --- /dev/null +++ b/scripts/import-pathway-data.ts @@ -0,0 +1,1747 @@ +/** + * Import pathway metadata from the working Excel workbook into v2 JSON (#860). + * + * Supersedes the one-shot `codemod-v1-to-v2.ts`: where the codemod scaffolded + * `coreDrivers` null, `dependencies` [], and a single widest-scope keyFeatures + * entry, this reads the real values out of `pathway_data_prepared.xlsx` and + * writes them onto the metadata files. + * + * npx ts-node --esm scripts/import-pathway-data.ts --dry-run + * npx ts-node --esm scripts/import-pathway-data.ts + * npx ts-node --esm scripts/import-pathway-data.ts --xlsx other.xlsx --sheet-prefix pathway_ + * + * Then run `npm run schema:check` and prettier over the written src/data JSON. + * + * This is a development tool, not a runtime path. It writes files in place and + * prints a report of everything it could not map cleanly (unresolved scopes, + * enum/vocabulary mismatches, over-length prose) rather than emitting anything + * that would fail `npm run schema:check`. + * + * Sheet names are parameterized (`--sheet-prefix`, default `draft_`) because the + * final `pathway_*` sheets are not populated yet; switch the prefix once they are. + * + * Scope of the current workbook: 24 pathways -- IEA (WEO 2025), ACE, UN SDSN/CW + * (Thailand, Myanmar, Laos, Singapore) and JETP-ID. Earlier versions also + * carried Philippines DOE and JRC; their TARGETS entries stay, and the report + * lists them as having no matching row. It carries NO NGFS data, so issue #801 + * (NGFS region memberships) is NOT addressed here -- the report says so. + */ +import { promises as fs } from "node:fs"; +import { fileURLToPath } from "node:url"; +import ExcelJS from "exceljs"; +import { + PATHWAY_DESCRIPTION, + splitExpertOverview, + upgradeV1ToV2, +} from "./codemod-v1-to-v2.ts"; +import type { PathwayMetadataV1 } from "../src/types/pathwayMetadata.v1.d.ts"; +import { + availabilityMetricsForSector, + technologyBelongsToSector, +} from "../src/utils/timeseriesTaxonomy.ts"; +import countryCodeSchema from "../src/schema/common/countryCode.v1.json" with { type: "json" }; +import pathwayMetadataV2Schema from "../src/schema/pathwayMetadata.v2.json" with { type: "json" }; +import sectorSchema from "../src/schema/common/sector.v1.json" with { type: "json" }; +import technologySchema from "../src/schema/common/technology.v1.json" with { type: "json" }; +import metricSchema from "../src/schema/common/metric.v1.json" with { type: "json" }; +import emissionsScopeSchema from "../src/schema/common/emissionsScope.v1.json" with { type: "json" }; +import dataAvailabilitySchema from "../src/schema/common/dataAvailability.v1.json" with { type: "json" }; +import dataAvailabilityMetricSchema from "../src/schema/common/dataAvailabilityMetric.v1.json" with { type: "json" }; +import sectorSegmentSchema from "../src/schema/common/sectorSegment.v1.json" with { type: "json" }; + +/** Valid ISO 3166-1 alpha-2 codes, from the schema (rejects e.g. "XK"). */ +const VALID_COUNTRIES = new Set( + (countryCodeSchema as { enum?: string[] }).enum ?? [], +); + +const V2_ID = + (pathwayMetadataV2Schema as { $id?: string }).$id ?? + "http://pathways.rmi.org/schema/pathwayMetadata.v2.json"; + +/** Widest sentinels, mirroring src/utils/validateScopes.ts. */ +const ACROSS_SECTORS = "across sectors"; +const GLOBAL_SCOPE = "Global"; +const ACROSS_REGIONS = "across regions"; + +/** + * Workbook sheet selection -- the single source of truth. The data is not + * finalized, so this is expected to change: + * - `SHEET_PREFIX` picks which set to read. It currently reads the populated + * `draft_` sheets; change it to `"pathway_"` (or override for one run with + * `--sheet-prefix pathway_`) once the final sheets are filled in. + * - `SHEET_BASENAMES` maps each logical sheet to its un-prefixed name; edit + * here if a sheet is renamed. + * - The `draft_` sheets carry two header rows (row 1 is a prose instruction); + * the final `pathway_` sheets carry one. `TWO_HEADER_ROW_PREFIX` selects that. + */ +const SHEET_PREFIX = "draft_"; +const TWO_HEADER_ROW_PREFIX = "draft_"; +const SHEET_BASENAMES = { + metadata: "metadata", + keyFeatures: "key_features", + coreDrivers: "core_drivers", + dependencies: "dependencies", + dataAvailability: "data_availability", +} as const; + +// -------------------------------------------------------------------------- +// Controlled vocabularies, read from src/schema/** at load time so they cannot +// drift from the schema this output must validate against. Only the Excel-to- +// schema *mappings* below are literals -- those have no schema representation. +// -------------------------------------------------------------------------- + +/** Just enough of the JSON Schema shape to pull enums out of. */ +type SchemaNode = { + enum?: string[]; + type?: string; + $ref?: string; + anyOf?: SchemaNode[]; + items?: SchemaNode; + properties?: Record; + $defs?: Record; + definitions?: Record; +}; + +/** The `displayName` enum of a common vocabulary schema (sector/tech/metric). */ +function displayNameEnum(schema: SchemaNode): string[] { + return (schema.$defs ?? schema.definitions ?? {}).displayName?.enum ?? []; +} + +const SECTORS = displayNameEnum(sectorSchema as SchemaNode); +const METRICS = displayNameEnum(metricSchema as SchemaNode); +const TECHNOLOGIES = displayNameEnum(technologySchema as SchemaNode); + +/** dataAvailability vocabularies (#870), read from the schema like the rest. */ +const DA_DEFS = (dataAvailabilitySchema as SchemaNode).$defs ?? {}; +const DA_METRICS = displayNameEnum(dataAvailabilityMetricSchema as SchemaNode); +const SEGMENTS = displayNameEnum(sectorSegmentSchema as SchemaNode); +const TIME_RESOLUTIONS = DA_DEFS.timeResolution?.enum ?? []; +const DATA_FORMATS = DA_DEFS.dataFormat?.enum ?? []; +const GRANULARITY_VALUES = [ + ...(DA_DEFS.granularityBreakdown?.enum ?? []), + ...TECHNOLOGIES, +]; +/** Sentinels shared by the list-valued fields; never resolved as places. */ +const DA_SENTINELS = ["Unspecified", "Not covered"]; + +const V2_PROPS = (pathwayMetadataV2Schema as SchemaNode).properties ?? {}; +const PATHWAY_TYPES = V2_PROPS.pathwayType?.enum ?? []; +const EVIDENCE_TYPES = + V2_PROPS.dependencies?.items?.properties?.evidence_type?.enum ?? []; + +/** + * Per-field keyFeatures value enums, read straight from the v2 schema. A field's + * `value` takes one of four shapes: + * + * - an inline enum (most fields); + * - an array of enum strings (policyTypes / newTechnologiesIncluded); + * - a `$ref` to a shared enum schema; + * - an `anyOf` composing a `$ref` with an inline enum — emissionsScope, which + * draws its gas vocabulary from `emissionsScope.v1.json` (shared with + * pathwayTimeseries.v1) and adds the scope sentinel locally. + * + * The last shape is why `membersOf` is recursive rather than a branch per case. + * Returning `[]` for an unrecognised shape is what it used to do for an + * unhandled one, and it fails quietly: `matchEnum` then rejects every cell and + * the whole field is dropped from every pathway. Hence the assertion below. + * + * KF_ARRAY_FIELDS records which fields are arrays. + */ +const REF_ENUMS: Record = { + "emissionsScope.v1.json": (emissionsScopeSchema as SchemaNode).enum ?? [], +}; + +function membersOf(node: SchemaNode | undefined): readonly string[] { + if (!node) return []; + if (node.enum) return node.enum; + if (node.$ref) return REF_ENUMS[node.$ref.split("/").pop() ?? ""] ?? []; + if (node.anyOf) return node.anyOf.flatMap(membersOf); + return []; +} + +const KF_SCALAR_ENUMS: Record = {}; +const KF_ARRAY_ENUMS: Record = {}; +const KF_ARRAY_FIELDS = new Set(); +for (const [field, spec] of Object.entries( + V2_PROPS.keyFeatures?.properties ?? {}, +)) { + const value = spec.items?.properties?.value; + if (!value) continue; + if (value.type === "array") { + KF_ARRAY_ENUMS[field] = membersOf(value.items); + KF_ARRAY_FIELDS.add(field); + } else { + KF_SCALAR_ENUMS[field] = membersOf(value); + } +} + +// An empty vocabulary is never legitimate, and silently means "drop every value +// this field has". Better to refuse to run than to emit 60 files quietly +// missing a field. +for (const [field, members] of [ + ...Object.entries(KF_SCALAR_ENUMS), + ...Object.entries(KF_ARRAY_ENUMS), +]) { + if (members.length === 0) { + throw new Error( + `keyFeatures.${field}: could not read a value vocabulary out of the v2` + + ` schema. Extend membersOf() to cover its 'value' shape -- leaving it` + + ` empty would drop every ${field} cell in the workbook.`, + ); + } +} +for (const [name, members] of Object.entries({ + dataAvailabilityMetric: DA_METRICS, + sectorSegment: SEGMENTS, + timeResolution: TIME_RESOLUTIONS, + dataFormat: DATA_FORMATS, +})) { + if (members.length === 0) { + throw new Error( + `dataAvailability: the ${name} vocabulary read empty from the schema --` + + ` every availability row would be dropped.`, + ); + } +} + +/** Excel key_features column header -> v2 keyFeatures field name. */ +const KF_COLUMN_TO_FIELD: Record = { + "Emissions trajectory": "emissionsTrajectory", + "Energy efficiency": "energyEfficiency", + "Energy demand": "energyDemand", + "Electrification": "electrification", + "Policy types": "policyTypes", + "Technology cost trend": "technologyCostTrend", + "Emissions scope": "emissionsScope", + "Policy ambition": "policyAmbition", + "Technology costs detail": "technologyCostsDetail", + "New technologies included": "newTechnologiesIncluded", + "Investment needs": "investmentNeeds", +}; + +/** Excel core_drivers column header -> v2 coreDrivers field name. */ +const CORE_DRIVER_COLUMNS: Array<[string, string]> = [ + ["Policy", "policies"], + ["Emissions targets", "emissionsTargets"], + ["Technology costs", "technologyCosts"], + ["Investment trend", "investmentChange"], + ["Macroeconomic trends", "macroeconomicDrivers"], + ["Behavioral shifts", "behavioralShifts"], + ["Other drivers", "otherDrivers"], +]; + +/** Excel dependency dimension column -> schema `dependency_name` enum value. */ +const DEPENDENCY_COLUMNS: Array<[string, string]> = [ + ["Policy Strategy", "Policy strategy"], + ["Regulatory framework", "Regulatory framework"], + ["Market and economics", "Market and economics"], + ["Public acceptance", "Public acceptance"], + ["Consumer and client behavior", "Consumer and client behavior"], + ["Infrastructure and logistics", "Infrastructure and logistics"], + ["Technology", "Technology"], + ["Resource availability", "Resource availability"], + [ + "Environmental impacts and ecosystem services", + "Environmental impacts and ecosystem services", + ], + ["Labor availability", "Labor availability"], +]; + +/** + * Geography tokens that keyFeatures rows use but the metadata `Regions` cell + * spells differently. Only applied when the canonical form is a geography the + * pathway actually declares, so it can never widen coverage. + */ +const GEO_ALIASES: Record = { + "eu": "European Union", + "european union": "EU", +}; + +// -------------------------------------------------------------------------- +// Row -> target file mapping, keyed as canonicalKey() builds it: the only +// stable join key across sheets is (publisher group, scenario code, +// country), since pathway-name strings and publisher labels vary between +// sheets (e.g. "State" vs "States" in ATS). IEA is a new WEO edition (2025 vs +// the repo's 2024, and CPS is a brand-new scenario) so its rows create new +// files and leave the 2024 files untouched, per the "keep both" decision. +// Everyone else maps 1:1 onto an existing file and is updated in place. +// -------------------------------------------------------------------------- + +type Target = + | { mode: "update"; file: string; wasV1?: boolean; note?: string } + | { + mode: "new"; + file: string; + id: string; + templateFile: string; + note?: string; + }; + +export const TARGETS: Record = { + // IEA World Energy Outlook 2025 -- new edition, new files. + "IEA:CPS:": { + mode: "new", + file: "src/data/iea/IEA-CPS-2025.json", + id: "IEA-CPS-2025", + templateFile: "src/data/iea/IEA-STEPS-2024.json", + note: "new WEO-2025 scenario (no 2024 equivalent)", + }, + "IEA:STEPS:": { + mode: "new", + file: "src/data/iea/IEA-STEPS-2025.json", + id: "IEA-STEPS-2025", + templateFile: "src/data/iea/IEA-STEPS-2024.json", + note: "WEO-2025 edition; IEA-STEPS-2024 kept", + }, + "IEA:NZE:": { + mode: "new", + file: "src/data/iea/IEA-NZE-2025.json", + id: "IEA-NZE-2025", + templateFile: "src/data/iea/IEA-NZE-2024.json", + note: "WEO-2025 edition; IEA-NZE-2024 kept", + }, + // ACE 8th ASEAN Energy Outlook -- already v2, refreshed. + "ACE:BAS:": { + mode: "update", + file: "src/data/asean-centre-for-energy/ACE-BAS-2024.json", + }, + "ACE:ATS:": { + mode: "update", + file: "src/data/asean-centre-for-energy/ACE-ATS-2024.json", + }, + "ACE:RAS:": { + mode: "update", + file: "src/data/asean-centre-for-energy/ACE-RAS-2024.json", + }, + "ACE:CNS:": { + mode: "update", + file: "src/data/asean-centre-for-energy/ACE-CNS-2024.json", + }, + // UN SDSN / ClimateWorks -- Thailand. + "SDSN:EXT TH:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-EXT-TH-2024.json", + wasV1: true, + }, + "SDSN:MAP TH:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-MAP-TH-2024.json", + wasV1: true, + }, + "SDSN:NZE TH:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-NZE-TH-2024.json", + wasV1: true, + }, + // Philippines DOE. Excel Year of Publication is 2024 while the repo ids say + // 2023 (id uses the plan's base year); same single report, so update in place. + "PHDOE:REF:": { + mode: "update", + file: "src/data/philippines-department-of-energy/PHDOE-REFERENCE-2023.json", + wasV1: true, + note: "Excel year 2024 vs repo id year 2023 (same report)", + }, + "PHDOE:CES1:": { + mode: "update", + file: "src/data/philippines-department-of-energy/PHDOE-CES1-2023.json", + wasV1: true, + note: "Excel year 2024 vs repo id year 2023 (same report)", + }, + "PHDOE:CES2:": { + mode: "update", + file: "src/data/philippines-department-of-energy/PHDOE-CES2-2023.json", + wasV1: true, + note: "Excel year 2024 vs repo id year 2023 (same report)", + }, + // ASEAN Green Future -- stored under un-sdsn-cw. The workbook now publishes + // these as "UN SDSN, CW" with the country inside the code ("EPP MM"), not as + // "ASEAN Green Future ... [MM]", so the keys follow that spelling. Under the + // old keys all seven silently stopped matching. + "SDSN:EPP MM:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-EPP-MM-2025.json", + wasV1: true, + }, + "SDSN:OEPP MM:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-OEPP-MM-2025.json", + wasV1: true, + }, + "SDSN:OMAP MM:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-OMAP-MM-2025.json", + wasV1: true, + }, + "SDSN:EP LA:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-EP-LA-2025.json", + wasV1: true, + }, + "SDSN:OEP LA:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-OEP-LA-2025.json", + wasV1: true, + }, + "SDSN:MAP LA:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-MAP-LA-2025.json", + wasV1: true, + }, + "SDSN:OMAP LA:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-OMAP-LA-2025.json", + wasV1: true, + }, + // UN SDSN / ClimateWorks -- Singapore. Existing v1 files, migrated in place. + "SDSN:BAS SG:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-BAS-SG-2024.json", + wasV1: true, + }, + "SDSN:BAU SG:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-BAU-SG-2024.json", + wasV1: true, + }, + "SDSN:HA1S SG:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-HA1S-SG-2024.json", + wasV1: true, + }, + "SDSN:HA2S SG:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-HA2S-SG-2024.json", + wasV1: true, + }, + "SDSN:HA1O SG:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-HA1O-SG-2024.json", + wasV1: true, + }, + "SDSN:HA2O SG:": { + mode: "update", + file: "src/data/un-sdsn-cw/SDSN-CW-HA2O-SG-2024.json", + wasV1: true, + }, + // JRC GECO -- repo files already year 2025, same edition; update in place. + "JRC:REFERENCE:": { + mode: "update", + file: "src/data/jrc/JRC-GECO-REFERENCE-2025.json", + wasV1: true, + }, + "JRC:NDC-LTS:": { + mode: "update", + file: "src/data/jrc/JRC-GECO-NDC-LTS-2025.json", + wasV1: true, + }, + "JRC:1.5C:": { + mode: "update", + file: "src/data/jrc/JRC-GECO-1-5-2025.json", + wasV1: true, + }, + // JETP Indonesia. + "JETP:CIPP:": { + mode: "update", + file: "src/data/jetp-id/JETP-CIPP-2023.json", + wasV1: true, + }, +}; + +/** Publisher label (any sheet's spelling) -> canonical group used in the key. */ +export function publisherGroup(label: string): string | null { + const l = label.toLowerCase(); + if (l.includes("green future") || l.includes("agf")) return "AGF"; + if (l.includes("iea") || l.includes("international energy")) return "IEA"; + if (l.includes("asean centre") || l === "ace") return "ACE"; + if (l.includes("philippines") || l.includes("department of energy")) + return "PHDOE"; + if (l.includes("joint research") || l.includes("jrc")) return "JRC"; + if (l.includes("jetp")) return "JETP"; + if (l.includes("sdsn") || l.includes("climateworks")) return "SDSN"; + return null; +} + +/** Scenario code + optional [CC] country, from the pathway name. */ +export function scenarioParts(name: string): { code: string; country: string } { + const country = /\[([A-Z]{2})\]/.exec(name)?.[1] ?? ""; + const parens = [...name.matchAll(/\(([^)]+)\)/g)].map((m) => m[1]); + let code = parens.length > 0 ? parens[parens.length - 1] : ""; + if (!code) { + // JRC pathways have no parenthetical code. + const l = name.toLowerCase(); + if (l.startsWith("reference")) code = "REFERENCE"; + else if (l.includes("ndc-lts")) code = "NDC-LTS"; + else if (l.includes("1.5")) code = "1.5C"; + } + return { code, country }; +} + +export function canonicalKey( + publisher: string, + pathwayName: string, +): string | null { + const group = publisherGroup(publisher); + if (!group) return null; + const { code, country } = scenarioParts(pathwayName); + if (!code) return null; + return `${group}:${code}:${country}`; +} + +// -------------------------------------------------------------------------- +// Workbook reading. +// -------------------------------------------------------------------------- + +type Row = Record; + +interface Sheets { + metadata: Row[]; + keyFeatures: Row[]; + coreDrivers: Row[]; + dependencies: Row[]; + dataAvailability: Row[]; +} + +/** Flatten one ExcelJS cell value to trimmed text. */ +function cellText(value: ExcelJS.CellValue): string { + if (value === null || value === undefined) return ""; + if (typeof value === "string") return value.trim(); + if (typeof value === "number" || typeof value === "boolean") + return String(value); + if (value instanceof Date) return String(value.getFullYear()); + if (typeof value === "object") { + const v = value as Record; + if (Array.isArray(v.richText)) { + return v.richText + .map((r) => (r as { text?: string }).text ?? "") + .join("") + .trim(); + } + if ("text" in v && typeof v.text === "string") return v.text.trim(); + if ("result" in v) return cellText(v.result as ExcelJS.CellValue); + if ("formula" in v) return ""; + } + return String(value).trim(); +} + +/** + * Read one sheet into records. Draft sheets have two header rows -- row 1 is a + * prose instruction, row 2 is the field name -- so field names come from row 2 + * and data from row 3 on. `CHECK *` and blank-header helper columns are dropped. + * The final `pathway_*` sheets carry a single header row; `headerRow` handles that. + */ +function readSheet( + ws: ExcelJS.Worksheet | undefined, + headerRow: number, +): Row[] { + if (!ws) return []; + const headers: Record = {}; + ws.getRow(headerRow).eachCell({ includeEmpty: true }, (cell, col) => { + const h = cellText(cell.value); + if (h && !h.startsWith("CHECK")) headers[col] = h; + }); + const rows: Row[] = []; + for (let r = headerRow + 1; r <= ws.rowCount; r++) { + const row = ws.getRow(r); + const rec: Row = {}; + let any = false; + for (const [colStr, header] of Object.entries(headers)) { + const text = cellText(row.getCell(Number(colStr)).value); + rec[header] = text; + if (text) any = true; + } + if (any) rows.push(rec); + } + return rows; +} + +async function readWorkbook(path: string, prefix: string): Promise { + const wb = new ExcelJS.Workbook(); + await wb.xlsx.readFile(path); + // Draft sheets: two header rows. Final `pathway_` sheets: one. + const headerRow = prefix === TWO_HEADER_ROW_PREFIX ? 2 : 1; + const sheet = (base: string) => wb.getWorksheet(`${prefix}${base}`); + return { + metadata: readSheet(sheet(SHEET_BASENAMES.metadata), headerRow), + keyFeatures: readSheet(sheet(SHEET_BASENAMES.keyFeatures), headerRow), + coreDrivers: readSheet(sheet(SHEET_BASENAMES.coreDrivers), headerRow), + dependencies: readSheet(sheet(SHEET_BASENAMES.dependencies), headerRow), + dataAvailability: readSheet( + sheet(SHEET_BASENAMES.dataAvailability), + headerRow, + ), + }; +} + +// -------------------------------------------------------------------------- +// Value normalization helpers. +// -------------------------------------------------------------------------- + +/** Cells that mean "no value at this scope" -- the entry is omitted entirely. */ +/** + * Whether a cell records nothing at all, as against recording an absence. + * + * The `not applicable` / `not available` prefixes used to be here, and taking + * them out is the point of the change. Cookbook decision 0023 made those + * authored values rather than empty cells: `Not applicable at this scope level` + * says this row is not the widest scope the field is calculated for, and + * `Not Applicable` says the measure is meaningless for the sector at any scope. + * #858 added both to the schema, and the epic turns on the distinction — an + * explicit value terminates #869's fallback chain where an absent entry keeps + * broadening. Swallowing them here threw away exactly what they were added to + * express. + * + * A consequence worth keeping: a cell still carrying a pre-0023 spelling now + * fails `matchEnum` and is reported in `badValues` instead of vanishing, so the + * importer's own report names the cells left to correct. + */ +export function isAbsent(raw: string): boolean { + const l = raw.trim().toLowerCase(); + return l === "" || l === "null" || l === "n/a"; +} + +export function matchEnum( + raw: string, + options: readonly string[], +): string | null { + const cleaned = raw.trim().replace(/signifcant/gi, "Significant"); + const hit = options.find((o) => o.toLowerCase() === cleaned.toLowerCase()); + return hit ?? null; +} + +export function ensurePeriod(text: string): string { + const t = text.trim(); + return /\.$/.test(t) ? t : `${t}.`; +} + +export function normalizeSector(raw: string): string | null { + return ( + SECTORS.find((s) => s.toLowerCase() === raw.trim().toLowerCase()) ?? null + ); +} + +function normalizeTechnology(raw: string): string | null { + return ( + TECHNOLOGIES.find((t) => t.toLowerCase() === raw.trim().toLowerCase()) ?? + null + ); +} + +function normalizeMetric(raw: string): string | null { + return ( + METRICS.find((m) => m.toLowerCase() === raw.trim().toLowerCase()) ?? null + ); +} + +/** Split a delimited list cell on `;` and `,`, dropping empties. */ +export function splitList(raw: string): string[] { + return raw + .split(/[;,]/) + .map((s) => s.trim()) + .filter((s) => s.length > 0); +} + +// -------------------------------------------------------------------------- +// Reporting. +// -------------------------------------------------------------------------- + +export class Report { + readonly lines: string[] = []; + /** + * Authoring mistakes the workbook has to fix. Unlike a note, any one of these + * stops the import: main() writes no files while the list is non-empty. + */ + readonly errors: string[] = []; + section(title: string) { + this.lines.push(`\n## ${title}`); + } + note(text: string) { + this.lines.push(` - ${text}`); + } + error(text: string) { + this.errors.push(text); + } +} + +// -------------------------------------------------------------------------- +// Field builders. +// -------------------------------------------------------------------------- + +interface Geography { + global?: boolean; + regions?: Record; + country?: string[]; +} + +/** + * Parse the metadata `Regions` cell into a v2 geography object. Format: + * Global; Africa: [DZ, EG]; Atlantic Basin: []; Brazil: [BR]; PH + * A bare token (`Global`) sets the flag or, if a 2-letter code, becomes a + * standalone country. `Label: [..]` is a region and its ISO members. + */ +export function buildGeography( + cell: string, + report: Report, + id: string, +): Geography | null { + const raw = cell.trim().replace(/^"+|"+$/g, ""); + if (!raw) return null; + const geo: Geography = {}; + const regions: Record = {}; + const countries: string[] = []; + // Split top-level entries on `;` (member lists inside [..] use commas only). + for (const part of raw + .split(";") + .map((s) => s.trim()) + .filter(Boolean)) { + const m = /^(.+?):\s*\[(.*)\]$/.exec(part); + if (m) { + const label = m[1].trim(); + const all = m[2] + .split(",") + .map((s) => s.trim().toUpperCase()) + .filter((s) => /^[A-Z]{2}$/.test(s)); + const members = all.filter((c) => VALID_COUNTRIES.has(c)); + const dropped = all.filter((c) => !VALID_COUNTRIES.has(c)); + if (dropped.length) { + report.note( + `${id}: geography region "${label}" dropped non-ISO code(s) {${dropped.join(", ")}}`, + ); + } + // `LA: [LA]` is how the cookbook writes an individually projected country + // (validator rule R4.3 flags the bare `LA` form as non-standard), not a + // region that happens to be named after its only member. Reading it as a + // region made every single-country pathway a pseudo-region -- except + // Thailand, whose cell is still written bare. + const code = label.toUpperCase(); + if ( + /^[A-Z]{2}$/.test(code) && + VALID_COUNTRIES.has(code) && + members.length === 1 && + members[0] === code + ) { + countries.push(code); + continue; + } + regions[label] = members; + } else if (/^global$/i.test(part)) { + geo.global = true; + } else if (/^[A-Z]{2}$/.test(part.toUpperCase()) && part.length === 2) { + const c = part.toUpperCase(); + if (VALID_COUNTRIES.has(c)) countries.push(c); + else report.note(`${id}: geography dropped non-ISO country "${part}"`); + } else { + // A region label with no bracketed member list. + regions[part] = []; + report.note( + `${id}: region "${part}" declared with no members (matches nothing)`, + ); + } + } + if (Object.keys(regions).length > 0) geo.regions = regions; + if (countries.length > 0) geo.country = countries; + return geo; +} + +/** Every geography token entries on this pathway may name (mirrors validateScopes). */ +export function allowedGeographies(geo: Geography | undefined): Set { + const allowed = new Set([ACROSS_REGIONS]); + if (!geo) return allowed; + if (geo.global === true) allowed.add(GLOBAL_SCOPE); + for (const [label, members] of Object.entries(geo.regions ?? {})) { + allowed.add(label); + members.forEach((m) => allowed.add(m)); + } + (geo.country ?? []).forEach((c) => allowed.add(c)); + return allowed; +} + +/** Alnum-only, lowercase -- for punctuation/spacing-insensitive label matching. */ +function normKey(s: string): string { + return s.toLowerCase().replace(/[^a-z0-9]/g, ""); +} + +export function resolveGeography( + token: string, + allowed: Set, +): string | null { + const t = token.trim(); + if (allowed.has(t)) return t; + const up = t.toUpperCase(); + if (/^[A-Z]{2}$/.test(up) && allowed.has(up)) return up; + const alias = GEO_ALIASES[t.toLowerCase()]; + if (alias && allowed.has(alias)) return alias; + // Last resort: match ignoring case, punctuation and spacing, so a key-feature + // token like "Rest Sub Saharan Africa" resolves to a declared region labeled + // "Rest Sub-Saharan Africa". Genuine word differences ("Other" vs "Rest") do + // not match and are reported instead. + const tn = normKey(t); + for (const a of allowed) { + if (normKey(a) === tn) return a; + } + return null; +} + +function sectorBreadth(s: string): number { + return s === ACROSS_SECTORS ? 0 : 1; +} +function geographyBreadth(g: string): number { + if (g === GLOBAL_SCOPE) return 0; + if (g === ACROSS_REGIONS) return 1; + return /^[A-Z]{2}$/.test(g) ? 3 : 2; +} + +type ScopedEntry = { + sector: string; + geography: string; + value: string | string[]; +}; + +/** + * Build the 11 scoped keyFeatures arrays from this pathway's key_features rows. + * Emits every resolvable (sector, geography) scope. Entries whose sector or + * geography does not resolve against the pathway's declared coverage are + * omitted and reported, so the file still passes validateScopedEntries. + */ +function buildKeyFeatures( + rows: Row[], + declaredSectors: Set, + geoAllowed: Set, + report: Report, + id: string, +): Record { + const fields: Record = {}; + const seen: Record> = {}; + for (const field of Object.values(KF_COLUMN_TO_FIELD)) { + fields[field] = []; + seen[field] = new Set(); + } + const badSectors = new Set(); + const badGeos = new Set(); + const badValues = new Set(); + let dupScopes = 0; + + for (const row of rows) { + const rawSector = row["Key features sector scope"] ?? ""; + const rawGeo = row["Key features regional scope"] ?? ""; + /* + The sheet's sector-scope sentinel is the schema's, so this normalizes + case and nothing else. It used to translate `across sectors` into a + repo-local `cross-sector`; #858 settled on the cookbook's spelling, so + that translation now produces a value the schema rejects -- and did so for + 40 of this workbook's 98 key-feature rows. + */ + let sector: string | null; + if (rawSector.trim().toLowerCase() === ACROSS_SECTORS) + sector = ACROSS_SECTORS; + else sector = normalizeSector(rawSector); + if ( + !sector || + (sector !== ACROSS_SECTORS && !declaredSectors.has(sector)) + ) { + badSectors.add(rawSector.trim() || "(blank)"); + continue; + } + const geography = resolveGeography(rawGeo, geoAllowed); + if (!geography) { + badGeos.add(rawGeo.trim() || "(blank)"); + continue; + } + const scopeKey = `${sector}\u0000${geography}`; + + for (const [column, field] of Object.entries(KF_COLUMN_TO_FIELD)) { + const raw = row[column] ?? ""; + if (isAbsent(raw)) continue; + let value: string | string[] | null = null; + if (KF_ARRAY_FIELDS.has(field)) { + const tokens = splitList(raw); + const mapped: string[] = []; + for (const tok of tokens) { + const hit = matchEnum(tok, KF_ARRAY_ENUMS[field]); + if (hit) { + if (!mapped.includes(hit)) mapped.push(hit); + } else { + badValues.add(`${field}:"${tok}"`); + } + } + value = mapped.length > 0 ? mapped : null; + } else { + const hit = matchEnum(raw, KF_SCALAR_ENUMS[field]); + if (hit) value = hit; + else badValues.add(`${field}:"${raw}"`); + } + if (value === null) continue; + if (seen[field].has(scopeKey)) { + dupScopes++; + continue; + } + seen[field].add(scopeKey); + fields[field].push({ sector, geography, value }); + } + } + + const emitted = Object.values(fields).reduce((n, a) => n + a.length, 0); + const parts: string[] = [`${emitted} keyFeature entries emitted`]; + if (badSectors.size) + parts.push(`undeclared sector scopes {${[...badSectors].join(", ")}}`); + if (badGeos.size) + parts.push(`unresolved geographies {${[...badGeos].join(", ")}}`); + // Named, not just counted: these are the cells to correct in the workbook, + // and a bare count says nothing about which. `badSectors` and `badGeos` above + // have always named theirs; this one is now the more useful of the three, + // since isAbsent stopped hiding retired scope-sentinel spellings from it. + if (badValues.size) + parts.push( + `${badValues.size} off-enum value(s) dropped {${[...badValues].join(", ")}}`, + ); + if (dupScopes) parts.push(`${dupScopes} duplicate-scope value(s) collapsed`); + if (parts.length > 1) report.note(`${id}: ${parts.join("; ")}`); + + for (const field of Object.keys(fields)) { + fields[field].sort( + (a, b) => + sectorBreadth(a.sector) - sectorBreadth(b.sector) || + geographyBreadth(a.geography) - geographyBreadth(b.geography) || + a.sector.localeCompare(b.sector) || + a.geography.localeCompare(b.geography), + ); + } + return fields; +} + +function buildCoreDrivers( + row: Row | undefined, + report: Report, + id: string, +): Record { + const out: Record = {}; + for (const [column, field] of CORE_DRIVER_COLUMNS) { + const raw = row?.[column] ?? ""; + if (isAbsent(raw)) { + out[field] = null; + continue; + } + const text = ensurePeriod(raw); + if (text.length > 500) { + report.note( + `${id}/coreDrivers.${field}: ${text.length} chars > 500 -- left null, needs trimming`, + ); + out[field] = null; + } else { + out[field] = text; + } + } + return out; +} + +function buildDependencies( + rows: Row[], + declaredSectors: Set, + report: Report, + id: string, +): Array> { + const deps: Array> = []; + const seen = new Set(); + for (const row of rows) { + const sector = normalizeSector(row["Sector"] ?? ""); + if (!sector) { + report.note( + `${id}/dependencies: sector "${row["Sector"]}" not a valid sector -- rows skipped`, + ); + continue; + } + if (!declaredSectors.has(sector)) { + report.note( + `${id}/dependencies: sector "${sector}" not declared by pathway -- rows skipped`, + ); + continue; + } + for (const [column, name] of DEPENDENCY_COLUMNS) { + const prose = row[column] ?? ""; + if (isAbsent(prose)) continue; + const description = ensurePeriod(prose); + if (description.length > 500) { + report.note( + `${id}/dependencies "${name}" (${sector}): ${description.length} chars > 500 -- skipped`, + ); + continue; + } + const evidence = + matchEnum(row[`${column}_Evidence type`] ?? "", EVIDENCE_TYPES) ?? + "No evidence"; + const key = `${name}\u0000${sector}\u0000${description}`; + if (seen.has(key)) continue; + seen.add(key); + deps.push({ + dependency_name: name, + dependency_description: description, + sector, + evidence_type: evidence, + }); + } + } + return deps; +} + +// -------------------------------------------------------------------------- +// dataAvailability (#870), from the data_availability sheet. +// -------------------------------------------------------------------------- + +type ByMetricRow = { + metricName: string; + sector: string; + sectorSegment: string[]; + geography: string[]; + timeResolution: string; + dataFormat: string; + granularity: string[]; + scopeLimitations: string; +}; + +/** + * The inner list of a `Sector: [a, b]` cell, for this row's sector. Cells also + * arrive bare (`a, b`, `a; b`), in which case they pass through unchanged. + */ +export function unwrapSectorGroup(cell: string, sector: string): string { + const groups = [...cell.matchAll(/([^:;[\]]+):\s*\[([^\]]*)\]/g)]; + if (groups.length === 0) return cell; + const own = groups.find( + (g) => g[1].trim().toLowerCase() === sector.toLowerCase(), + ); + return (own ?? groups[0])[2]; +} + +/** Segments, case-normalised against the schema; sentinels included. */ +export function parseSegments( + cell: string, + sector: string, + bad: Set, +): string[] { + const out: string[] = []; + for (const tok of unwrapSectorGroup(cell, sector).split(/[;,]/)) { + const t = tok.trim(); + if (!t) continue; + const hit = matchEnum(t, SEGMENTS); + if (hit) { + if (!out.includes(hit)) out.push(hit); + } else bad.add(`sectorSegment:"${t}"`); + } + return out; +} + +/** + * Geography coverage: `;`-separated, each token either a bare label (`ASEAN`, + * `TH`) or the cookbook's `Label: [members]` form, of which only the label is + * a scope token. Labels resolve against the pathway's own declared geography, + * the same way key-feature scopes do. + */ +export function parseCoverage( + cell: string, + geoAllowed: Set, + bad: Set, +): string[] { + const out: string[] = []; + for (const chunk of cell.split(";")) { + const label = chunk + .replace(/\[[^\]]*\]/g, "") + .replace(/:\s*$/, "") + .trim(); + if (!label) continue; + const hit = + matchEnum(label, DA_SENTINELS) ?? resolveGeography(label, geoAllowed); + if (hit) { + if (!out.includes(hit)) out.push(hit); + } else bad.add(`geography:"${label}"`); + } + return out; +} + +/** + * Granularity: `;` or ` / ` separated. Commas are not separators at the top + * level, because several values contain them ("Capital costs, O&M, etc.", + * "By sector, part of value chain"); a chunk is split on commas only when it is + * not itself a known value and every part is. + */ +export function parseGranularity(cell: string, bad: Set): string[] { + const out: string[] = []; + const add = (v: string) => { + if (!out.includes(v)) out.push(v); + }; + for (const chunk of cell.split(/\s*;\s*|\s+\/\s+/)) { + const t = chunk.trim(); + if (!t) continue; + const hit = matchEnum(t, GRANULARITY_VALUES); + if (hit) { + add(hit); + continue; + } + const parts = t + .split(",") + .map((x) => x.trim()) + .filter(Boolean); + const hits = parts.map((x) => matchEnum(x, GRANULARITY_VALUES)); + if (parts.length > 1 && hits.every(Boolean)) hits.forEach((h) => add(h!)); + else bad.add(`granularity:"${t}"`); + } + return out; +} + +/** + * The byMetric rows for one pathway. A row is emitted only when every required + * field is present: an empty Data Format, say, is unauthored data, and making + * one up would put a claim about the publication into the repo that nobody + * made. Skipped rows are named in the report -- they are the authoring worklist. + */ +export function buildDataAvailability( + rows: Row[], + declaredSectors: Set, + geoAllowed: Set, + report: Report, + id: string, +): { overall: null; byMetric: ByMetricRow[] } | null { + if (rows.length === 0) return null; + const byMetric: ByMetricRow[] = []; + const skipped: string[] = []; + const bad = new Set(); + const seen = new Set(); + + for (const row of rows) { + const rawMetric = (row["Metric"] ?? "").trim(); + const sector = normalizeSector(row["Sector"] ?? ""); + if (!sector || !declaredSectors.has(sector)) { + skipped.push( + `${rawMetric}: sector "${row["Sector"] ?? ""}" not declared`, + ); + continue; + } + // Sector-aware first: Power spells it "Absolute Emissions", Steel and + // Aviation "Absolute emissions", and a case-insensitive match against the + // flat list would hand Steel the Power spelling. + const metricName = + matchEnum(rawMetric, availabilityMetricsForSector(sector) ?? []) ?? + matchEnum(rawMetric, DA_METRICS); + const timeResolution = matchEnum( + row["Time resolution"] ?? "", + TIME_RESOLUTIONS, + ); + const dataFormat = matchEnum(row["Data Format"] ?? "", DATA_FORMATS); + const scopeLimitations = (row["Scope limitations"] ?? "").trim(); + const sectorSegment = parseSegments( + row["Sector segment"] ?? "", + sector, + bad, + ); + const geography = parseCoverage( + row["Geography coverage"] ?? "", + geoAllowed, + bad, + ); + const granularity = parseGranularity(row["Granularity"] ?? "", bad); + + // Every column is required (a row for every allowable pair, per the + // cookbook), so a blank or unrecognised cell is an authoring error that + // blocks the import -- dropping the row would silently lose what the rest + // of it says. + const missing = [ + !metricName && "metric", + !timeResolution && "time resolution", + !dataFormat && "data format", + !scopeLimitations && "scope limitations", + scopeLimitations.length > 500 && "scope limitations > 500 chars", + sectorSegment.length === 0 && "sector segment", + geography.length === 0 && "geography", + granularity.length === 0 && "granularity", + ].filter(Boolean); + if (missing.length > 0 || !metricName || !timeResolution || !dataFormat) { + report.error( + `${id}/dataAvailability ${sector} ${rawMetric || "(no metric)"}: ` + + `blank or unrecognised ${missing.join(", ")}`, + ); + continue; + } + + // "Not covered" describes the whole (sector, metric) pair, so a row either + // carries it on every variable or on none -- the rule validateScopedEntries + // enforces. A row authored as covered on some variables and "Not covered" + // on others contradicts itself, and only the author can say which half is + // right: report it as an error rather than guess. + const notCovered = (v: string | string[]) => + Array.isArray(v) ? v.includes("Not covered") : v === "Not covered"; + const variables = { + "sector segment": sectorSegment, + geography, + "time resolution": timeResolution, + "data format": dataFormat, + granularity, + "scope limitations": scopeLimitations, + }; + const uncovered = Object.entries(variables) + .filter(([, v]) => notCovered(v)) + .map(([k]) => k); + if (uncovered.length > 0 && uncovered.length < 6) { + report.error( + `${id}/dataAvailability ${sector} ${metricName}: "Not covered" on ` + + `${uncovered.join(", ")} only -- either every column says ` + + `"Not covered" or none does`, + ); + continue; + } + + const key = [ + metricName, + sector, + [...sectorSegment].sort().join(","), + [...geography].sort().join(","), + ].join("\u0000"); + if (seen.has(key)) { + skipped.push(`${metricName}: duplicate scope`); + continue; + } + seen.add(key); + byMetric.push({ + metricName, + sector, + sectorSegment, + geography, + timeResolution, + dataFormat, + granularity, + scopeLimitations, + }); + } + + const parts = [`${byMetric.length} availability row(s) emitted`]; + if (skipped.length) + parts.push(`${skipped.length} skipped {${skipped.join("; ")}}`); + if (bad.size) parts.push(`unrecognised values {${[...bad].join(", ")}}`); + report.note(`${id}: ${parts.join("; ")}`); + + return byMetric.length > 0 ? { overall: null, byMetric } : null; +} + +/** Parse `Sector: [t, t]; Sector2: [..]` into a sector -> raw-tech-tokens map. */ +function parseSectorGroups(cell: string): Record { + const out: Record = {}; + if (!cell) return out; + for (const part of cell + .split(";") + .map((s) => s.trim()) + .filter(Boolean)) { + const m = /^(.+?):\s*\[?([^\]]*)\]?$/.exec(part); + if (!m) continue; + out[m[1].trim()] = m[2] + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + } + return out; +} + +/** Build v2 `sectors` for a new file from the Excel Sectors + Technology coverage. */ +function buildSectors( + sectorsCell: string, + techCell: string, + report: Report, + id: string, +): Array<{ name: string; technologies: string[] }> { + const techGroups = parseSectorGroups(techCell); + // Normalize the tech-group keys to sector display names for lookup. + const techBySector: Record = {}; + for (const [k, v] of Object.entries(techGroups)) { + const s = normalizeSector(k); + if (s) techBySector[s] = v; + } + const out: Array<{ name: string; technologies: string[] }> = []; + const seen = new Set(); + for (const rawName of splitList(sectorsCell)) { + const name = normalizeSector(rawName); + if (!name) { + report.note(`${id}/sectors: "${rawName}" not a valid sector -- skipped`); + continue; + } + if (seen.has(name)) continue; + seen.add(name); + const technologies: string[] = []; + for (const rawTech of techBySector[name] ?? []) { + const tech = normalizeTechnology(rawTech); + if (tech && technologyBelongsToSector(tech, name) === "yes") { + if (!technologies.includes(tech)) technologies.push(tech); + } else if (rawTech) { + report.note( + `${id}/sectors.${name}: technology "${rawTech}" not valid for sector -- dropped`, + ); + } + } + out.push({ name, technologies }); + } + return out; +} + +/** Build the flat `metric` array for a new file from the Excel Metric cell. */ +function buildMetrics( + metricCell: string, + report: Report, + id: string, +): string[] { + const groups = parseSectorGroups(metricCell); + const metrics: string[] = []; + for (const tokens of Object.values(groups)) { + for (const raw of tokens) { + const m = normalizeMetric(raw); + if (m) { + if (!metrics.includes(m)) metrics.push(m); + } else if (raw) { + report.note( + `${id}/metric: "${raw}" not in metric vocabulary -- dropped`, + ); + } + } + } + return metrics; +} + +function parseYear(raw: string): number | undefined { + const n = Number(raw); + return Number.isInteger(n) && n >= 1900 && n <= 2100 ? n : undefined; +} + +function parseTemp(raw: string): number | undefined { + const n = Number(raw); + if (!Number.isFinite(n) || n < 0.5 || n > 3) return undefined; + return Math.round(n * 10) / 10; +} + +// -------------------------------------------------------------------------- +// Document assembly. +// -------------------------------------------------------------------------- + +type Json = Record; + +/** Assemble the full v2 document for one pathway. */ +function buildDoc( + target: Target, + meta: Row, + sub: { + keyFeatures: Row[]; + coreDrivers: Row | undefined; + dependencies: Row[]; + dataAvailability: Row[]; + }, + existing: Json | null, + template: Json | null, + report: Report, +): Json { + const id = + target.mode === "new" ? target.id : String(existing?.id ?? target.file); + + // Geography: rebuilt from Excel where present, else keep existing. + let geography = buildGeography( + meta["Regions"] ?? "", + report, + id, + ) as Geography | null; + if (!geography) + geography = + (existing?.geography as Geography) ?? + (template?.geography as Geography) ?? + null; + else if (existing?.geography) { + const before = JSON.stringify(existing.geography); + const after = JSON.stringify(geography); + if (before !== after) { + report.note( + `${id}: geography rebuilt. before regions=[${Object.keys((existing.geography as Geography).regions ?? {}).join(", ")}]` + + ` country=[${((existing.geography as Geography).country ?? []).join(", ")}] global=${(existing.geography as Geography).global}` + + ` -> after regions=[${Object.keys(geography.regions ?? {}).join(", ")}]` + + ` country=[${(geography.country ?? []).join(", ")}] global=${geography.global}`, + ); + } + } + + // Sectors are rebuilt from the Excel for every file, not preserved: the + // workbook's Sectors cell is authoritative (it drives which keyFeatures scopes + // are in-bounds), and preserving a v1 file's sectors could carry a technology + // that violates #461 and fails schema:check. Technologies are filtered to the + // sector's own vocabulary (technologiesForSector), dropping e.g. "BESS". + const sectors = buildSectors( + meta["Sectors"] ?? "", + meta["Technology coverage"] ?? "", + report, + id, + ); + if (sectors.length === 0 && existing?.sectors) { + report.note( + `${id}: Excel declared no parseable sectors -- kept existing sectors`, + ); + } + const effectiveSectors = + sectors.length > 0 + ? sectors + : ((existing?.sectors as typeof sectors) ?? []); + const declaredSectors = new Set(effectiveSectors.map((s) => s.name)); + + // Metric: rebuilt from Excel for new files; preserved for updates (existing + // values are already valid and tie to the hosted timeseries). + const metric = + target.mode === "new" + ? buildMetrics(meta["Metric"] ?? "", report, id) + : ((existing?.metric as string[]) ?? []); + + const geoAllowed = allowedGeographies(geography ?? undefined); + const fromV1 = + sub.keyFeatures.length === 0 + ? keyFeaturesFromV1(existing, report, id) + : null; + if (sub.keyFeatures.length === 0 && !fromV1) + report.note(`${id}: no key_features rows -- keyFeatures left empty`); + const keyFeatures = + fromV1 ?? + buildKeyFeatures(sub.keyFeatures, declaredSectors, geoAllowed, report, id); + const coreDrivers = buildCoreDrivers(sub.coreDrivers, report, id); + const dependencies = buildDependencies( + sub.dependencies, + declaredSectors, + report, + id, + ); + // Rebuilt from the sheet when it has rows for this pathway; otherwise + // whatever the file already carries is kept. + const dataAvailability = + buildDataAvailability( + sub.dataAvailability, + declaredSectors, + geoAllowed, + report, + id, + ) ?? (existing?.dataAvailability as Json | undefined); + + // pathwayDescription from the Excel "Pathway Overview"; keep existing otherwise. + let pathwayDescription: string | null; + const overview = (meta["Pathway Overview"] ?? "").trim(); + if (overview && !isAbsent(overview)) { + const text = ensurePeriod(overview); + if (text.length <= 2500) pathwayDescription = text; + else { + report.note( + `${id}/pathwayDescription: ${text.length} chars > 2500 -- kept existing`, + ); + pathwayDescription = (existing?.pathwayDescription as string) ?? null; + } + } else { + pathwayDescription = + (existing?.pathwayDescription as string | null) ?? + descriptionFromV1(existing, report, id); + } + + // Scalars: take from Excel where valid, else keep existing/template. + const pathwayType = (PATHWAY_TYPES as readonly string[]).includes( + meta["Pathway Type"] ?? "", + ) + ? meta["Pathway Type"] + : (existing?.pathwayType ?? template?.pathwayType); + const modelYearStart = + parseYear(meta["Start Year of Model"] ?? "") ?? + (existing?.modelYearStart as number | undefined) ?? + (template?.modelYearStart as number | undefined); + const modelYearEnd = + parseYear(meta["End Year of Model"] ?? "") ?? + (existing?.modelYearEnd as number | undefined) ?? + (template?.modelYearEnd as number | undefined); + const netzero = parseYear(meta["Net Zero Reached"] ?? ""); + const modelYearNetzero = + (netzero && netzero >= 2030 ? netzero : undefined) ?? + (existing?.modelYearNetzero as number | undefined); + const modelTempIncrease = + parseTemp(meta["Modeled Temperature Increase"] ?? "") ?? + (existing?.modelTempIncrease as number | undefined) ?? + (template?.modelTempIncrease as number | undefined); + + // Publication / name / description. + let publication: unknown; + let name: unknown; + let description: unknown; + if (target.mode === "new") { + const { code } = scenarioParts(meta["Name of Pathway"] ?? ""); + const full = (meta["Name of Pathway"] ?? "") + .replace(/\s*\([^)]*\)/g, "") + .replace(/\s*\[[A-Z]{2}\]/g, "") + .trim(); + name = { full, short: code }; + const desc = ensurePeriod(meta["Description"] ?? ""); + description = + desc.length <= 100 + ? desc + : (template?.description ?? desc.slice(0, 99) + "."); + if (desc.length > 100) + report.note( + `${id}/description: ${desc.length} chars > 100 -- used template description`, + ); + publication = buildPublicationForNew( + template?.publication as Json, + meta, + report, + id, + ); + } else { + publication = existing?.publication; + name = existing?.name; + description = existing?.description; + } + + const doc: Json = { + $schema: V2_ID, + id, + publication, + name, + description, + geography, + pathwayType, + }; + if (modelTempIncrease !== undefined) + doc.modelTempIncrease = modelTempIncrease; + if (modelYearStart !== undefined) doc.modelYearStart = modelYearStart; + if (modelYearEnd !== undefined) doc.modelYearEnd = modelYearEnd; + if (modelYearNetzero !== undefined) doc.modelYearNetzero = modelYearNetzero; + doc.sectors = effectiveSectors; + doc.pathwayDescription = pathwayDescription; + doc.metric = metric; + doc.keyFeatures = keyFeatures; + doc.coreDrivers = coreDrivers; + doc.dependencies = dependencies; + if (dataAvailability) doc.dataAvailability = dataAvailability; + return doc; +} + +/** + * A v1 file's description, for when the workbook has none. + * + * The fallback used to be `existing.pathwayDescription` alone. A v1 document has + * no such field -- its description sits inside `expertOverview` -- so migrating a + * v1 file whose workbook row had an empty "Pathway Overview" set the description + * to null and dropped the v1 text. That is how 13 pathways (3 JRC, 3 PHDOE, 7 + * Myanmar/Laos) lost 4-5 KB of description each. + * + * Extracts the "Pathway Description" section with the codemod's own parser, so + * the importer and the codemod cannot disagree about where it is. + */ +export function descriptionFromV1( + existing: Json | null, + report: Report, + id: string, +): string | null { + const overview = existing?.expertOverview; + if (typeof overview !== "string" || overview.trim() === "") return null; + const section = splitExpertOverview(overview).get(PATHWAY_DESCRIPTION) ?? ""; + if (section.trim() === "") return null; + const text = ensurePeriod(section); + if (text.length > 2500) { + report.note( + `${id}/pathwayDescription: v1 description is ${text.length} chars > 2500 -- left null`, + ); + return null; + } + report.note(`${id}/pathwayDescription: taken from v1 expertOverview`); + return text; +} + +/** + * The v1 key features of an existing file, converted to v2 scoped entries. + * + * A pathway with no key_features rows used to get 11 empty arrays, which on a + * v1 file throws away every authored value -- how PHDOE-CES1 and + * PHDOE-REFERENCE came to show "No information" throughout. Converting with the + * codemod's own upgradeV1ToV2 keeps the two migrations from disagreeing about + * how a v1 value is scoped. + */ +export function keyFeaturesFromV1( + existing: Json | null, + report: Report, + id: string, +): Record | null { + if (!existing || existing.$schema === V2_ID || !existing.keyFeatures) + return null; + const { doc } = upgradeV1ToV2(existing as unknown as PathwayMetadataV1); + report.note(`${id}/keyFeatures: no key_features rows -- converted from v1`); + return doc.keyFeatures as unknown as Record; +} + +/** Publication block for a new file: template's publisher block + Excel title/year. */ +function buildPublicationForNew( + template: Json | undefined, + meta: Row, + report: Report, + id: string, +): Json { + const pub: Json = template ? JSON.parse(JSON.stringify(template)) : {}; + const title = (meta["Name of publication"] ?? "").trim(); + const year = parseYear(meta["Year of Publication"] ?? ""); + const oldYear = pub.year as number | undefined; + if (title) pub.title = { ...(pub.title as Json), full: title }; + if (year !== undefined) pub.year = year; + // Bump any year in the link URLs to match (WEO 2024 -> 2025). + if (oldYear && year && Array.isArray(pub.links)) { + pub.links = ( + pub.links as Array<{ url?: string; description?: string }> + ).map((l) => ({ + ...l, + url: l.url?.replace(String(oldYear), String(year)), + })); + } + if (!template) + report.note( + `${id}: no template for publication -- block may be incomplete`, + ); + return pub; +} + +// -------------------------------------------------------------------------- +// Main. +// -------------------------------------------------------------------------- + +async function readJson(file: string): Promise { + try { + return JSON.parse(await fs.readFile(file, "utf8")) as Json; + } catch { + return null; + } +} + +/** Group sub-sheet rows by canonical key. */ +function groupByKey(rows: Row[]): Map { + const map = new Map(); + for (const row of rows) { + const key = canonicalKey( + row["Name of Publisher"] ?? "", + row["Name of Pathway"] ?? "", + ); + if (!key) continue; + (map.get(key) ?? map.set(key, []).get(key)!).push(row); + } + return map; +} + +async function main() { + const args = process.argv.slice(2); + const dryRun = args.includes("--dry-run"); + const getArg = (flag: string, def: string) => { + const i = args.indexOf(flag); + return i >= 0 && args[i + 1] ? args[i + 1] : def; + }; + const xlsx = getArg("--xlsx", "pathway_data_prepared.xlsx"); + const prefix = getArg("--sheet-prefix", SHEET_PREFIX); + const outRoot = getArg("--out", "src/data"); + void outRoot; // targets carry explicit paths; --out reserved for future use. + + const report = new Report(); + const sheets = await readWorkbook(xlsx, prefix); + report.note( + `Read ${xlsx} sheets ${prefix}{metadata,key_features,core_drivers,dependencies,data_availability}: ` + + `${sheets.metadata.length} pathways, ${sheets.keyFeatures.length} key-feature rows, ` + + `${sheets.dataAvailability.length} data-availability rows.`, + ); + + const kfByKey = groupByKey(sheets.keyFeatures); + const cdByKey = groupByKey(sheets.coreDrivers); + const depByKey = groupByKey(sheets.dependencies); + const daByKey = groupByKey(sheets.dataAvailability); + + const seenKeys = new Set(); + let updated = 0; + let created = 0; + let migratedV1 = 0; + const unmatched: string[] = []; + // Written only once every pathway has been built without an error, so a + // workbook with mistakes leaves src/data untouched rather than half-imported. + const pending: { file: string; doc: Json }[] = []; + + report.section("Per-pathway"); + for (const meta of sheets.metadata) { + const publisher = meta["Name of Publisher"] ?? ""; + const pathway = meta["Name of Pathway"] ?? ""; + const key = canonicalKey(publisher, pathway); + if (!key || !TARGETS[key]) { + unmatched.push(`${publisher} | ${pathway} (key=${key ?? "?"})`); + continue; + } + seenKeys.add(key); + const target = TARGETS[key]; + const existing = + target.mode === "update" ? await readJson(target.file) : null; + const template = + target.mode === "new" ? await readJson(target.templateFile) : null; + if (target.mode === "update" && !existing) { + report.note(`${key}: target file ${target.file} not found -- skipped`); + continue; + } + + const sub = { + keyFeatures: kfByKey.get(key) ?? [], + coreDrivers: (cdByKey.get(key) ?? [])[0], + dependencies: depByKey.get(key) ?? [], + dataAvailability: daByKey.get(key) ?? [], + }; + + const doc = buildDoc(target, meta, sub, existing, template, report); + const wasV1 = target.mode === "update" && existing?.$schema !== V2_ID; + + pending.push({ file: target.file, doc }); + if (target.mode === "new") { + created++; + report.note( + `NEW ${target.file}${target.note ? ` (${target.note})` : ""}`, + ); + } else { + updated++; + if (wasV1) migratedV1++; + report.note( + `UPD ${target.file}${wasV1 ? " [v1->v2]" : ""}${target.note ? ` (${target.note})` : ""}`, + ); + } + } + + // Targets in the table that no metadata row hit. + for (const [key, t] of Object.entries(TARGETS)) { + if (!seenKeys.has(key)) + report.note(`table entry ${key} -> ${t.file} had no matching Excel row`); + } + + report.section("Summary"); + report.note( + `${dryRun ? "Would update" : "Updated"} ${updated} file(s) (${migratedV1} migrated v1->v2), ` + + `${dryRun ? "would create" : "created"} ${created} new file(s).`, + ); + if (unmatched.length > 0) { + report.note( + `Unmatched Excel rows (${unmatched.length}): ${unmatched.join(" ;; ")}`, + ); + } + report.note( + "#801 NOT addressed: the workbook carries no NGFS data, so the 7 NGFS files' " + + "empty region memberships are untouched. Region memberships ARE populated for " + + "the covered pathways from their Regions cell.", + ); + + console.info(report.lines.join("\n")); + + if (report.errors.length > 0) { + console.error( + `\n## ${report.errors.length} authoring error(s) -- nothing written\n` + + "Fix these in the workbook and re-run the import:\n" + + report.errors.map((e) => ` - ${e}`).join("\n"), + ); + process.exitCode = 1; + return; + } + if (dryRun) return; + for (const { file, doc } of pending) + await fs.writeFile(file, `${JSON.stringify(doc, null, 2)}\n`); + console.info( + '\nRun `npm run schema:check` and `npx prettier --write "src/data/**/*.json"` next.', + ); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main().catch((e: unknown) => { + console.error(String(e instanceof Error ? e.stack : e)); + process.exit(1); + }); +} diff --git a/scripts/schema-check-files.ts b/scripts/schema-check-files.ts index d079d859..f9115e64 100644 --- a/scripts/schema-check-files.ts +++ b/scripts/schema-check-files.ts @@ -5,8 +5,20 @@ import type { FileEntry } from "../src/utils/validateData.ts"; import { validateFilesBySchema } from "../src/utils/validateData.ts"; import { decideIncludeInvalid } from "../src/utils/loadData.ts"; import pathwayMetadata from "../src/schema/pathwayMetadata.v1.json" with { type: "json" }; +import pathwayMetadataV2 from "../src/schema/pathwayMetadata.v2.json" with { type: "json" }; import pathwayTimeseries from "../src/schema/pathwayTimeseries.v1.json" with { type: "json" }; +import pathwayTimeseriesV2 from "../src/schema/pathwayTimeseries.v2.json" with { type: "json" }; import { commonSchemas } from "../src/schema/common/index.ts"; +import type { PathwayMetadataV2 } from "../src/types/pathwayMetadata.v2.d.ts"; +import type { PathwayTimeseriesV2 } from "../src/types/pathwayTimeseries.v2.d.ts"; +import { + PATHWAY_METADATA_V2_ID, + validateScopedEntries, +} from "../src/utils/validateScopes.ts"; +import { + PATHWAY_TIMESERIES_V2_ID, + validateTimeseries, +} from "../src/utils/validateTimeseries.ts"; async function run(dir: string) { async function getJsonFilesRecursive(base: string): Promise { @@ -34,10 +46,58 @@ async function run(dir: string) { const { valid, invalid } = validateFilesBySchema(entries, [ pathwayMetadata, + pathwayMetadataV2, pathwayTimeseries, + pathwayTimeseriesV2, ...commonSchemas, ]); - return { dir, validCount: valid.length, invalid }; + + // Second pass over the AJV-valid v2 documents for the cross-field scope + // constraint draft-07 cannot express — see src/utils/validateScopes.ts. A + // document that fails here is reported exactly like a schema failure, so a + // mistyped region label breaks the build instead of silently matching nothing. + const scopeProblems = valid + .filter((r) => r.schemaId === PATHWAY_METADATA_V2_ID) + .map((r) => ({ + name: r.name, + errors: validateScopedEntries(r.data as PathwayMetadataV2), + })) + .filter((p) => p.errors.length > 0); + + // Timeseries rows name a geography and segments that must match the + // pathway's metadata, which lives in other files -- see + // src/utils/validateTimeseries.ts. Metadata of either version counts: the + // geography object has the same shape in both. + const METADATA_IDS = new Set([ + String(pathwayMetadata.$id), + PATHWAY_METADATA_V2_ID, + ]); + const metadataGeographyById = new Map( + valid + .filter((r) => METADATA_IDS.has(r.schemaId)) + .map((r) => { + const doc = r.data as PathwayMetadataV2; + return [doc.id, doc.geography] as const; + }), + ); + const timeseriesProblems = valid + .filter((r) => r.schemaId === PATHWAY_TIMESERIES_V2_ID) + .map((r) => ({ + name: r.name, + errors: validateTimeseries( + r.data as PathwayTimeseriesV2, + metadataGeographyById, + ), + })) + .filter((p) => p.errors.length > 0); + + const crossFileProblems = [...scopeProblems, ...timeseriesProblems]; + const badNames = new Set(crossFileProblems.map((p) => p.name)); + return { + dir, + validCount: valid.filter((r) => !badNames.has(r.name)).length, + invalid: [...invalid, ...crossFileProblems], + }; } async function main() { @@ -73,8 +133,12 @@ async function main() { filesWithIssues += r.invalid.length; for (const p of r.invalid) { - // Emit up to N errors per file as GH annotations - const file = join(r.dir, p.name); + // Emit up to N errors per file as GH annotations. + // `p.name` already carries the directory — getJsonFilesRecursive builds it + // with join(base, d.name) — so re-prepending r.dir produced paths like + // "src/data/src/data/foo.json" and the annotations pointed nowhere. Latent + // until now because it only shows when a file actually fails. + const file = p.name; const errs = p.errors.slice(0, 50); for (const e of errs) { console.log(`::${annotate} file=${file}::${e}`); diff --git a/scripts/timeseries-csv.test.ts b/scripts/timeseries-csv.test.ts new file mode 100644 index 00000000..4dbc05d3 --- /dev/null +++ b/scripts/timeseries-csv.test.ts @@ -0,0 +1,49 @@ +import { describe, it, expect } from "vitest"; +import { csvCell, jsonToCsv, sectorSegmentColumns } from "./timeseries-csv.ts"; + +describe("sectorSegmentColumns", () => { + it("lists the row's segments and their definitions in the same order", () => { + const cols = sectorSegmentColumns("power", [ + "Power generation", + "Energy storage", + ]); + expect(cols.sector_segment).toBe("Power generation; Energy storage"); + const definitions = cols.definition_sector_segment.split("; "); + expect(definitions).toHaveLength(2); + expect(definitions[0]).toMatch(/Generation of electricity/); + expect(definitions[1]).toMatch(/Storing electricity/); + }); + + it("leaves both cells empty for a row with no segments", () => { + expect(sectorSegmentColumns("power", undefined)).toEqual({ + sector_segment: "", + definition_sector_segment: "", + }); + }); + + it("throws on a segment the sector does not define", () => { + expect(() => sectorSegmentColumns("power", ["Ironmaking"])).toThrow( + /Unknown segment "Ironmaking"/, + ); + }); +}); + +describe("CSV writing", () => { + it("quotes strings and joins a list into one quoted cell", () => { + expect(csvCell('say "hi"')).toBe('"say ""hi"""'); + expect(csvCell(["Power generation", "Energy storage"])).toBe( + '"Power generation; Energy storage"', + ); + expect(csvCell(42)).toBe("42"); + expect(csvCell(null)).toBe(""); + }); + + it("keeps a list in its own column", () => { + const csv = jsonToCsv([ + { geography: "Southeast Asia", sector_segment: ["A", "B"], value: 1 }, + ]); + const [header, row] = csv.split("\n"); + expect(header).toBe("geography,sector_segment,value"); + expect(row).toBe('"Southeast Asia","A; B",1'); + }); +}); diff --git a/scripts/timeseries-csv.ts b/scripts/timeseries-csv.ts new file mode 100644 index 00000000..fbe89249 --- /dev/null +++ b/scripts/timeseries-csv.ts @@ -0,0 +1,61 @@ +/** + * CSV pieces of the timeseries download, kept out of + * build-timeseries-files.ts so they can be tested: that script runs on import. + */ +import { + getSectorDefinition, + segmentDefinitionFor, + UnknownTaxonomyError, +} from "../src/utils/timeseriesTaxonomy.ts"; + +/** How a list value is written into one CSV cell. */ +export const LIST_SEPARATOR = "; "; + +/** + * The download's segment columns for one row: the segments it covers and, in + * the same order, what each means. + * + * They replace the former `sector_scope` column, which repeated one taxonomy + * string per metric; segments now come from the row itself (#915). A row + * without segments (a v1 file) gets empty cells rather than a guess. + */ +export function sectorSegmentColumns( + sectorKey: string, + segments: readonly string[] | undefined, +): { sector_segment: string; definition_sector_segment: string } { + const list = segments ?? []; + const sector = getSectorDefinition(sectorKey).displayName; + const definitions = list.map((segment) => { + const def = segmentDefinitionFor(sector, segment); + // Schema validation guarantees this; failing loudly beats a blank cell. + if (!def) + throw new UnknownTaxonomyError( + `Unknown segment "${segment}" for sector "${sector}"`, + ); + return def.definition; + }); + return { + sector_segment: list.join(LIST_SEPARATOR), + definition_sector_segment: definitions.join(LIST_SEPARATOR), + }; +} + +/** + * One CSV cell. Strings are always quoted; a list is joined first, so it can + * never spill its commas into the next columns. + */ +export function csvCell(value: unknown): string { + if (Array.isArray(value)) return csvCell(value.join(LIST_SEPARATOR)); + if (typeof value === "string") return `"${value.replace(/"/g, '""')}"`; + if (value === null || value === undefined) return ""; + return String(value); +} + +/** Rows to CSV text; the header is the union of the rows' keys. */ +export function jsonToCsv(data: readonly Record[]): string { + if (data.length === 0) return ""; + const keys = Array.from(new Set(data.flatMap((row) => Object.keys(row)))); + const header = keys.join(","); + const rows = data.map((row) => keys.map((k) => csvCell(row[k])).join(",")); + return [header, ...rows].join("\n"); +} diff --git a/src/components/Badge.test.tsx b/src/components/Badge.test.tsx index da5d2e75..35125abd 100644 --- a/src/components/Badge.test.tsx +++ b/src/components/Badge.test.tsx @@ -1,8 +1,29 @@ import { describe, it, expect } from "vitest"; import { render, screen } from "@testing-library/react"; +import { readFileSync } from "node:fs"; import Badge, { BadgeMaybeAbsent } from "./Badge"; import { pathwayTypeTooltips, sectorTooltips } from "../utils/tooltipUtils"; +/** Every variant the union offers, so the guard below cannot miss one. */ +const ALL_VARIANTS = [ + "default", + "pathwayType", + "temperature", + "year", + "geographyGlobal", + "geographyRegion", + "geographyCountry", + "sector", + "sectorSegment", + "metric", + "keyFeature", + "geographyGlobal-pub", + "geographyRegion-pub", + "geographyCountry-pub", + "sector-pub", + "metric-pub", +] as const; + describe("Badge component", () => { it("renders with the provided text", () => { render(Test Badge); @@ -131,6 +152,18 @@ describe("Badge component", () => { expect(badge).toHaveClass("border-solar-200"); }); + it("applies sectorSegment styling when variant is 'sectorSegment'", () => { + const { container } = render( + Power generation, + ); + const badge = container.firstChild as HTMLElement; + expect(badge).toHaveClass("bg-rmipink-100"); + expect(badge).toHaveClass("text-rmipink-800"); + expect(badge).toHaveClass("border-rmipink-200"); + // Must not be confusable with the sector badge it sits next to. + expect(badge).not.toHaveClass("bg-solar-100"); + }); + it("applies metric styling when variant is 'metric'", () => { const { container } = render(Intensity); const badge = container.firstChild as HTMLElement; @@ -156,8 +189,8 @@ describe("Badge component", () => { ); const badge = container.firstChild as HTMLElement; expect(badge).toHaveClass("bg-transparent"); - expect(badge).toHaveClass("text-pinishgreen-700"); - expect(badge).toHaveClass("border-pinishgreen-500"); + expect(badge).toHaveClass("text-pinishgreen-800"); + expect(badge).toHaveClass("border-pinishgreen-400"); }); it("applies geographyCountry-pub styling with transparent background", () => { @@ -166,7 +199,7 @@ describe("Badge component", () => { ); const badge = container.firstChild as HTMLElement; expect(badge).toHaveClass("bg-transparent"); - expect(badge).toHaveClass("text-pinishgreen-600"); + expect(badge).toHaveClass("text-pinishgreen-800"); expect(badge).toHaveClass("border-pinishgreen-400"); }); @@ -189,6 +222,51 @@ describe("Badge component", () => { }); }); + describe("every colour class resolves to a defined theme token", () => { + /* + Tailwind v4 takes its palette from `@theme` in index.css. A class naming a + shade that is not defined there emits no CSS at all and fails silently — an + outlined badge with no text colour and no border, which is exactly how + geographyRegion-pub and geographyCountry-pub shipped referencing + pinishgreen-500/600/700 when the family defines only 100/200/400/800. + + Asserting the class strings one by one cannot catch that, because the class + is present either way. This checks them against the theme. + */ + // Read rather than imported: vitest's CSS handling returns "" for + // `index.css?raw`, which would make every case below vacuously pass. The + // sanity check below stops that happening silently again. + const themeCss = readFileSync("src/index.css", "utf8"); + const defined = new Set( + [...themeCss.matchAll(/--color-([a-z0-9]+-[0-9]+)\s*:/g)].map( + (m) => m[1], + ), + ); + + // The families index.css owns. Anything else (transparent, white, and + // Tailwind's own built-ins) is not ours to verify. + const OWNED = + /^(?:bg|text|border)-((?:rmi[a-z]+|pinishgreen|solar)-[0-9]+)$/; + + it("has a non-empty theme to check against", () => { + // Guards the regex above: an empty set would make every case below vacuous. + expect(defined.size).toBeGreaterThan(20); + expect(defined).toContain("pinishgreen-800"); + }); + + it.each(ALL_VARIANTS)("%s", (variant) => { + const { container } = render(Label); + const badge = container.firstChild as HTMLElement; + + const missing = [...badge.classList] + .map((cls) => OWNED.exec(cls)?.[1]) + .filter((token): token is string => token !== undefined) + .filter((token) => !defined.has(token)); + + expect(missing).toEqual([]); + }); + }); + it("always includes base badge styling", () => { // Testing that common styles are applied to all variants const { container } = render( diff --git a/src/components/Badge.tsx b/src/components/Badge.tsx index b4c6e142..e35a4070 100644 --- a/src/components/Badge.tsx +++ b/src/components/Badge.tsx @@ -16,6 +16,7 @@ interface BadgeProps { | "geographyRegion" | "geographyCountry" | "sector" + | "sectorSegment" | "metric" | "keyFeature" | "geographyGlobal-pub" @@ -24,6 +25,14 @@ interface BadgeProps { | "sector-pub" | "metric-pub"; className?: string; + /** + * When set, the badge renders as a ` + ) : ( + {children} + ); } // With tooltip, use the TextWithTooltip component @@ -84,6 +111,8 @@ const Badge: React.FC = ({ text={{children}} tooltip={tooltip} position="right" + as={buttonProps ? "button" : "span"} + buttonProps={buttonProps} /> ); }; diff --git a/src/components/BadgeArray.test.tsx b/src/components/BadgeArray.test.tsx index 48461477..adee7849 100644 --- a/src/components/BadgeArray.test.tsx +++ b/src/components/BadgeArray.test.tsx @@ -1,5 +1,6 @@ -import { describe, it, expect } from "vitest"; -import { render, screen, waitFor } from "@testing-library/react"; +import { describe, it, expect, vi } from "vitest"; +import { render, screen, waitFor, fireEvent } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import BadgeArray from "./BadgeArray"; describe("BadgeArray", () => { @@ -59,12 +60,25 @@ describe("BadgeArray", () => { expect(screen.getByText("+3 more")).toBeInTheDocument(); }); + it("labels the hidden items behind '+N more' with toLabel too", async () => { + render( + + visibleCount={1} + toLabel={(v) => `Label ${v}`} + > + {["A", "B", "C"]} + , + ); + fireEvent.focus(screen.getByText("+2 more").closest("[tabindex]")!); + expect(await screen.findByRole("tooltip")).toHaveTextContent( + "Label B, Label C", + ); + }); + it("throws when variant array length does not match children length", () => { expect(() => render( - - {["Power", "Aviation"]} - , + {["Power", "Aviation"]}, ), ).toThrow(/length must match/); }); @@ -300,3 +314,128 @@ describe("auto-fit & maxRows", () => { expect(token).toBeTruthy(); }); }); + +describe("BadgeArray single-select toggles", () => { + it("renders static spans and no buttons without onSelect", () => { + // The regression guard for the four existing callers: opting out must + // leave the markup exactly as it was. + const { container } = render({["Power", "Steel"]}); + + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + expect(container.querySelectorAll("span").length).toBeGreaterThan(0); + }); + + it("renders each badge as a toggle button when onSelect is given", () => { + render( + {}} + visibleCount={Infinity} + > + {["Power", "Steel"]} + , + ); + + expect(screen.getAllByRole("button")).toHaveLength(2); + for (const button of screen.getAllByRole("button")) { + expect(button).toHaveAttribute("aria-pressed", "false"); + } + }); + + it("marks the selected item pressed", () => { + render( + {}} + visibleCount={Infinity} + > + {["Power", "Steel"]} + , + ); + + expect(screen.getByRole("button", { name: "Power" })).toHaveAttribute( + "aria-pressed", + "false", + ); + expect(screen.getByRole("button", { name: "Steel" })).toHaveAttribute( + "aria-pressed", + "true", + ); + }); + + it("reports the clicked item", async () => { + const onSelect = vi.fn(); + render( + + {["Power", "Steel"]} + , + ); + + await userEvent.click(screen.getByRole("button", { name: "Steel" })); + expect(onSelect).toHaveBeenCalledWith("Steel"); + }); + + it("reports null when the already-selected item is clicked", async () => { + const onSelect = vi.fn(); + render( + + {["Power", "Steel"]} + , + ); + + // Re-clicking the pressed badge is how an axis is cleared. + await userEvent.click(screen.getByRole("button", { name: "Steel" })); + expect(onSelect).toHaveBeenCalledWith(null); + }); + + it("keeps focus on the clicked token when the caller reorders", () => { + // The scope ribbon pins the selected token first, so the array order + // changes between renders. Index keys would rewrite nodes in place and + // leave focus on whatever token moved into that slot. + const { rerender } = render( + {}} + visibleCount={Infinity} + > + {["Power", "Steel", "Cement"]} + , + ); + + const steel = screen.getByRole("button", { name: "Steel" }); + steel.focus(); + expect(steel).toHaveFocus(); + + rerender( + {}} + visibleCount={Infinity} + > + {["Steel", "Power", "Cement"]} + , + ); + + expect(screen.getByRole("button", { name: "Steel" })).toHaveFocus(); + }); + + it("still renders a tooltip on a toggle badge", async () => { + render( + {}} + tooltipGetter={(item) => `About ${item}`} + visibleCount={Infinity} + > + {["Power"]} + , + ); + + fireEvent.focus(screen.getByRole("button")); + expect(await screen.findByRole("tooltip")).toHaveTextContent("About Power"); + }); +}); diff --git a/src/components/BadgeArray.tsx b/src/components/BadgeArray.tsx index 59c6ac10..33c25841 100644 --- a/src/components/BadgeArray.tsx +++ b/src/components/BadgeArray.tsx @@ -8,12 +8,14 @@ import React, { import Badge, { BadgeMaybeAbsent } from "./Badge"; import TextWithTooltip from "./TextWithTooltip"; -type Variant = React.ComponentProps["variant"]; +/** The badge variants, exported so callers can annotate a computed variant list. */ +export type BadgeVariant = React.ComponentProps["variant"]; +type Variant = BadgeVariant; type Scalar = string | number; export type BadgeArrayProps = Omit< React.ComponentProps, - "tooltip" + "tooltip" | "children" | "variant" | "toLabel" | "renderLabel" > & { /** Provide an array of scalar-ish values as children: {['A', 'B', null, 42]} */ children: Array; @@ -23,12 +25,24 @@ export type BadgeArrayProps = Omit< visibleCount?: number; /** Provide original item for tooltip content; still receives the raw item. */ tooltipGetter?: (item: T) => React.ReactNode; - /** Optionally map each raw item to a display label (allows non-scalars). */ - toLabel?: (item: T | null | undefined) => React.ReactNode; + /** + * Optionally map each raw item to a display label. Returns a string because + * the label is handed to BadgeMaybeAbsent, which renders it into a Badge + * (whose children are scalar) and may pass it through `renderLabel`. + */ + toLabel?: (item: T | null | undefined) => string; /** Optionally post-process a mapped label (e.g., highlight). */ renderLabel?: (label: React.ReactNode) => React.ReactNode; /** Max rows to display before collapsing into “+N more”. Use Infinity for unlimited rows. Default: 1. */ maxRows?: number; + /** + * Turns the badges into single-select toggles. Pass `onSelect` to opt in; + * `selected` marks which item reads as pressed. Without `onSelect` the markup + * is unchanged, so existing callers are unaffected. + */ + selected?: T | null; + /** Receives the clicked item, or null when it was already selected. */ + onSelect?: (next: T | null) => void; }; export default function BadgeArray({ @@ -39,6 +53,9 @@ export default function BadgeArray({ toLabel, renderLabel, maxRows = 1, + selected, + onSelect, + className, ...rest }: BadgeArrayProps) { // Memoize to keep stable reference across renders and satisfy hooks deps. @@ -230,23 +247,55 @@ export default function BadgeArray({ > +0 more - {arr.slice(0, finalVisible).map((value, idx) => ( - (itemRefs.current[idx] = el)} - className="inline-block" - > - { + const isSelected = + onSelect != null && value != null && value === selected; + // Ring and weight, never colour: colour already encodes the badge's + // variant (a geography's kind, say), so reusing it for selection would + // overload one channel with two meanings. + const itemClassName = [ + className, + isSelected + ? "ring-2 ring-bluespruce ring-offset-1 ring-offset-white font-semibold" + : "", + ] + .filter(Boolean) + .join(" "); + + return ( + { + itemRefs.current[idx] = el; + }} + className="inline-block" > - {value} - - - ))} + + onSelect(isSelected ? null : (value as T)), + } + : undefined + } + {...rest} + > + {value} + + + ); + })} {showMore && ( ({ {arr.slice(finalVisible).map((value, idx) => ( {idx > 0 && ", "} - {value} + {/* The same label the visible badges show: a country + code reads as its name here too. */} + + {toLabel ? toLabel(value) : value} + ))} diff --git a/src/components/ColumnGeographySelect.test.tsx b/src/components/ColumnGeographySelect.test.tsx new file mode 100644 index 00000000..5de54258 --- /dev/null +++ b/src/components/ColumnGeographySelect.test.tsx @@ -0,0 +1,259 @@ +import { describe, it, expect, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import ColumnGeographySelect from "./ColumnGeographySelect"; +import type { ColumnGeographyOption } from "../utils/comparisonScope"; + +/** + * IEA's real shape in miniature: Global and Southeast Asia are the two its + * timeseries actually carries; the rest are declared by the publication only. + */ +const OPTIONS: ColumnGeographyOption[] = [ + { token: "Global", label: "Global", kind: "global", available: true }, + { + token: "Southeast Asia", + label: "Southeast Asia", + kind: "region", + available: true, + }, + { token: "Europe", label: "Europe", kind: "region", available: false }, + { + token: "US", + label: "United States of America", + kind: "country", + available: false, + }, +]; + +const setup = ( + props: Partial> = {}, +) => { + const onSelect = vi.fn(); + render( + , + ); + return { onSelect, user: userEvent.setup() }; +}; + +const trigger = () => + screen.getByRole("button", { + name: "Geography for Announced Pledges Scenario", + }); + +describe("ColumnGeographySelect", () => { + describe("trigger", () => { + it("shows the selected geography", () => { + setup(); + expect(trigger()).toHaveTextContent("Global"); + }); + + it("names the column it belongs to, since position is not accessible", () => { + setup(); + // The visible text is a geography, so the accessible name has to carry + // which pathway this control is for. + expect(trigger()).toBeInTheDocument(); + expect(trigger()).toHaveAttribute("aria-haspopup", "listbox"); + }); + + it("reports its collapsed state", () => { + setup(); + expect(trigger()).toHaveAttribute("aria-expanded", "false"); + }); + + it("prompts when nothing is selected yet", () => { + setup({ selected: null }); + expect(trigger()).toHaveTextContent("Select a geography"); + }); + + it("is disabled and says so for a pathway declaring no geography", () => { + setup({ options: [], selected: null }); + expect(trigger()).toBeDisabled(); + expect(trigger()).toHaveTextContent("No geography"); + }); + }); + + describe("opening and closing", () => { + it("opens on click", async () => { + const { user } = setup(); + await user.click(trigger()); + + expect(screen.getByRole("listbox")).toBeInTheDocument(); + expect(trigger()).toHaveAttribute("aria-expanded", "true"); + }); + + it("closes on a second click of the trigger", async () => { + // The shell this borrows from opened without ever toggling closed. + const { user } = setup(); + await user.click(trigger()); + await user.click(trigger()); + + expect(screen.queryByRole("listbox")).toBeNull(); + }); + + it("closes on Escape, returning focus to the trigger", async () => { + const { user } = setup(); + await user.click(trigger()); + await user.keyboard("{Escape}"); + + expect(screen.queryByRole("listbox")).toBeNull(); + // Without this a keyboard reader loses their place in the page. + expect(trigger()).toHaveFocus(); + }); + + it("closes on a click outside", async () => { + const { user } = setup(); + await user.click(trigger()); + await user.click(document.body); + + expect(screen.queryByRole("listbox")).toBeNull(); + }); + }); + + describe("options", () => { + it("offers every declared geography, plainly labelled", async () => { + const { user } = setup(); + await user.click(trigger()); + + const names = screen + .getAllByRole("option") + .map((o) => o.textContent?.trim()); + expect(names).toEqual([ + "Global", + "Southeast Asia", + "Europe", + "United States of America", + ]); + // No publisher suffix: the column carries the provenance. + expect(names.join(" ")).not.toContain("("); + }); + + it("marks exactly one option selected", async () => { + const { user } = setup(); + await user.click(trigger()); + + const selected = screen + .getAllByRole("option") + .filter((o) => o.getAttribute("aria-selected") === "true"); + expect(selected.map((o) => o.textContent?.trim())).toEqual(["Global"]); + }); + + it("shows geographies this tool cannot plot, outlined rather than hidden", async () => { + // 13 of IEA's 15 declared geographies produce no chart. Hiding them would + // disagree with the coverage section, which lists all 15. + const { user } = setup(); + await user.click(trigger()); + + const europe = screen.getByRole("option", { name: "Europe" }); + expect(europe.querySelector(".bg-transparent")).not.toBeNull(); + + const global = screen.getByRole("option", { name: "Global" }); + expect(global.querySelector(".bg-transparent")).toBeNull(); + }); + + it("explains what the outlines mean", async () => { + const { user } = setup(); + await user.click(trigger()); + + expect( + screen.getByText( + /Outlined badges indicate the data for this geography/, + ), + ).toBeInTheDocument(); + }); + }); + + describe("selection", () => { + it("reports the token, not the label", async () => { + // The token is what lands in the URL and what the column resolves. + const { onSelect, user } = setup(); + await user.click(trigger()); + await user.click( + screen.getByRole("option", { name: "United States of America" }), + ); + + expect(onSelect).toHaveBeenCalledWith("US"); + }); + + it("closes after choosing", async () => { + const { user } = setup(); + await user.click(trigger()); + await user.click(screen.getByRole("option", { name: "Europe" })); + + expect(screen.queryByRole("listbox")).toBeNull(); + }); + + it("lets an unavailable geography be chosen", async () => { + // Shaded is a caveat, not a veto — the fallback note under the chart + // then says what was shown instead. + const { onSelect, user } = setup(); + await user.click(trigger()); + await user.click(screen.getByRole("option", { name: "Europe" })); + + expect(onSelect).toHaveBeenCalledWith("Europe"); + }); + }); + + describe("keyboard", () => { + it("focuses the selected option on open, not the first", async () => { + const { user } = setup({ selected: "Europe" }); + await user.click(trigger()); + + expect(screen.getByRole("option", { name: "Europe" })).toHaveFocus(); + }); + + it("moves through the options with the arrow keys", async () => { + const { user } = setup(); + await user.click(trigger()); + expect(screen.getByRole("option", { name: "Global" })).toHaveFocus(); + + await user.keyboard("{ArrowDown}"); + expect( + screen.getByRole("option", { name: "Southeast Asia" }), + ).toHaveFocus(); + + await user.keyboard("{ArrowUp}"); + expect(screen.getByRole("option", { name: "Global" })).toHaveFocus(); + }); + + it("stops at the ends rather than wrapping", async () => { + // With 152 options the ends are a useful landmark. + const { user } = setup(); + await user.click(trigger()); + + await user.keyboard("{ArrowUp}"); + expect(screen.getByRole("option", { name: "Global" })).toHaveFocus(); + + await user.keyboard("{End}{ArrowDown}"); + expect( + screen.getByRole("option", { name: "United States of America" }), + ).toHaveFocus(); + }); + + it("jumps to the ends with Home and End", async () => { + const { user } = setup(); + await user.click(trigger()); + + await user.keyboard("{End}"); + expect( + screen.getByRole("option", { name: "United States of America" }), + ).toHaveFocus(); + + await user.keyboard("{Home}"); + expect(screen.getByRole("option", { name: "Global" })).toHaveFocus(); + }); + + it("selects the focused option with Enter", async () => { + const { onSelect, user } = setup(); + await user.click(trigger()); + await user.keyboard("{ArrowDown}{Enter}"); + + expect(onSelect).toHaveBeenCalledWith("Southeast Asia"); + }); + }); +}); diff --git a/src/components/ColumnGeographySelect.tsx b/src/components/ColumnGeographySelect.tsx new file mode 100644 index 00000000..378e0645 --- /dev/null +++ b/src/components/ColumnGeographySelect.tsx @@ -0,0 +1,170 @@ +import React, { useEffect, useRef } from "react"; +import { ChevronDown } from "lucide-react"; +import Badge from "./Badge"; +import { useDropdown } from "../hooks/useDropdown"; +import { geographyVariant } from "../utils/geographyUtils"; +import type { ColumnGeographyOption } from "../utils/comparisonScope"; +import { GEOGRAPHY_AVAILABILITY_TOOLTIP } from "../utils/timeseriesAvailability"; + +interface ColumnGeographySelectProps { + /** The pathway this column shows. Names the control for assistive tech. */ + pathwayName: string; + /** This pathway's own geographies, from `columnGeographyOptions`. */ + options: ColumnGeographyOption[]; + selected: string | null; + onSelect: (token: string) => void; +} + +/** + * The geography control for one comparison column. + * + * A dropdown rather than a badge row because the lists do not fit: IEA declares + * 15 geographies and NGFS declares 152 (Global, 8 regions and 143 countries). + * A single row collapses those behind "+N more", and expanding them would bury + * the page under badges. + * + * One control per column rather than one for the page, because geography tokens + * are publication-specific — across the loadable pathways no token is declared + * by more than one publisher, so a shared value would put every other column + * into a fallback. The column carries the provenance, which is what lets the + * options be labelled plainly. + * + * Options are the same badges the Geographies coverage section shows, outlined + * where this tool holds no timeseries data for them. That distinction is + * load-bearing here: 13 of IEA's 15 declared geographies produce no chart. + * + * Built on `useDropdown` — the same open/close, click-outside and Escape + * behaviour as the search page's filter dropdowns. + */ +export const ColumnGeographySelect: React.FC = ({ + pathwayName, + options, + selected, + onSelect, +}) => { + const { open, triggerRef, menuRef, toggle, close } = + useDropdown(); + const listRef = useRef(null); + + const current = options.find((option) => option.token === selected); + + /* + Focus the selected option when the panel opens, as a listbox should: with + 152 options, dropping focus at the top would leave a keyboard reader + arrowing through the list to find where they already are. + */ + useEffect(() => { + if (!open) return; + const list = listRef.current; + if (!list) return; + const buttons = [...list.querySelectorAll("button")]; + const target = + buttons.find((b) => b.getAttribute("aria-selected") === "true") ?? + buttons[0]; + target?.focus(); + }, [open]); + + const onListKeyDown = (event: React.KeyboardEvent): void => { + const list = listRef.current; + if (!list) return; + const buttons = [...list.querySelectorAll("button")]; + const index = buttons.indexOf(document.activeElement as HTMLButtonElement); + if (index === -1) return; + + const step: Record = { + ArrowDown: 1, + ArrowUp: -1, + }; + const delta = step[event.key]; + + if (delta !== undefined) { + event.preventDefault(); + // Clamped rather than wrapping: the ends of the list are a useful + // landmark when it is long. + buttons[ + Math.min(buttons.length - 1, Math.max(0, index + delta)) + ]?.focus(); + return; + } + if (event.key === "Home") { + event.preventDefault(); + buttons[0]?.focus(); + } + if (event.key === "End") { + event.preventDefault(); + buttons[buttons.length - 1]?.focus(); + } + }; + + return ( +
+ + + {open && ( +
+
+
+ {options.map((option) => { + const base = geographyVariant(option.kind); + return ( + + ); + })} +
+

+ {GEOGRAPHY_AVAILABILITY_TOOLTIP} +

+
+
+ )} +
+ ); +}; + +export default ColumnGeographySelect; diff --git a/src/components/ComparisonKeyFeatures.test.tsx b/src/components/ComparisonKeyFeatures.test.tsx new file mode 100644 index 00000000..98e3b341 --- /dev/null +++ b/src/components/ComparisonKeyFeatures.test.tsx @@ -0,0 +1,247 @@ +import { describe, it, expect } from "vitest"; +import { render, screen, within } from "@testing-library/react"; +import ComparisonKeyFeatures from "./ComparisonKeyFeatures"; +import { NOT_A_CORE_DRIVER_TEXT } from "./CoreDrivers"; +import type { CoreDrivers } from "./CoreDrivers"; +import type { PathwayMetadataType } from "../types"; + +/** v2 stores every key feature as scoped entries; widest scope is what renders. */ +const wide = (value: string | string[]) => [ + { sector: "cross-sector", geography: "Global", value }, +]; + +const KEY_FEATURES = { + policyAmbition: wide("High ambition policies"), + policyTypes: wide(["Carbon price"]), + emissionsTrajectory: wide("Moderate decrease"), + emissionsScope: wide("CO2"), + technologyCostTrend: wide("Decrease"), + technologyCostsDetail: wide("Total costs"), + newTechnologiesIncluded: wide(["CCUS"]), + investmentNeeds: wide("Total investment"), + energyDemand: wide("Low or no change"), + electrification: wide("Moderate increase"), + energyEfficiency: wide("Significant improvement"), +}; + +/** Every driver null — the state of every pathway file in the repo today. */ +const NO_DRIVERS: CoreDrivers = { + policies: null, + emissionsTargets: null, + technologyCosts: null, + investmentChange: null, + macroeconomicDrivers: null, + behavioralShifts: null, + otherDrivers: null, +}; + +const pathway = ( + id: string, + coreDrivers: Partial = {}, +): PathwayMetadataType => + ({ + id, + name: { full: `Pathway ${id}`, short: id }, + keyFeatures: KEY_FEATURES, + coreDrivers: { ...NO_DRIVERS, ...coreDrivers }, + }) as unknown as PathwayMetadataType; + +const renderGrid = (pathways: PathwayMetadataType[]) => + render(); + +const groupHeadings = () => + screen.getAllByRole("heading", { level: 3 }).map((h) => h.textContent); + +const rowLabels = () => + screen + .getAllByText( + (_, el) => + el?.tagName === "P" && + el.className.includes("uppercase") && + el.className.includes("font-semibold"), + ) + .map((el) => el.textContent); + +describe("ComparisonKeyFeatures", () => { + describe("groups", () => { + it("shows the five feature-bearing groups when no driver is described", () => { + // Every core driver is null in every pathway file today, so this is what + // the page actually renders. + renderGrid([pathway("a"), pathway("b")]); + + expect(groupHeadings()).toEqual([ + "Policies", + "Emissions", + "Technology", + "Investment", + "Energy System", + ]); + }); + + it("skips Other until one of its drivers is described", () => { + // "Other" carries three drivers and no features, so with nothing + // authored it would render as an empty heading bar. + renderGrid([pathway("a"), pathway("b")]); + expect(groupHeadings()).not.toContain("Other"); + }); + + it("shows Other once any of its drivers is described", () => { + renderGrid([ + pathway("a", { behavioralShifts: "Reduced car ownership." }), + pathway("b"), + ]); + + expect(groupHeadings()).toEqual([ + "Policies", + "Emissions", + "Technology", + "Investment", + "Energy System", + "Other", + ]); + }); + }); + + describe("driver rows", () => { + it("omits a driver no compared pathway describes", () => { + // All-null carries no comparative information — every column would read + // the same sentence. + renderGrid([pathway("a"), pathway("b")]); + + expect(rowLabels()).not.toContain("Policy drivers"); + expect(screen.queryByText(NOT_A_CORE_DRIVER_TEXT)).toBeNull(); + }); + + it("shows a driver one pathway describes, and says the other does not", () => { + renderGrid([ + pathway("a", { policies: "Carbon pricing across both sectors." }), + pathway("b"), + ]); + + expect(rowLabels()).toContain("Policy drivers"); + expect( + screen.getByText("Carbon pricing across both sectors."), + ).toBeInTheDocument(); + // The contrast is the point: B not having it is a real statement. + expect(screen.getAllByText(NOT_A_CORE_DRIVER_TEXT)).toHaveLength(1); + }); + + it("reads the driver from each pathway's own data", () => { + renderGrid([ + pathway("a", { policies: "A's policy story." }), + pathway("b", { policies: "B's policy story." }), + ]); + + expect(screen.getByText("A's policy story.")).toBeInTheDocument(); + expect(screen.getByText("B's policy story.")).toBeInTheDocument(); + expect(screen.queryByText(NOT_A_CORE_DRIVER_TEXT)).toBeNull(); + }); + + it("puts a group's drivers above its features", () => { + // Matching the detail page: the authored narrative reads first and the + // enumerated values qualify it. + renderGrid([pathway("a", { policies: "Policy prose." }), pathway("b")]); + + const labels = rowLabels(); + expect(labels.indexOf("Policy drivers")).toBeLessThan( + labels.indexOf("Policy ambition"), + ); + }); + + it("places a driver in its own theme's group", () => { + renderGrid([ + pathway("a", { technologyCosts: "Falling battery costs." }), + pathway("b"), + ]); + + const labels = rowLabels(); + // technologyCosts belongs to Technology, so it precedes that group's + // features and follows the Emissions group's. + expect(labels.indexOf("Emissions scope")).toBeLessThan( + labels.indexOf("Technology cost drivers"), + ); + expect(labels.indexOf("Technology cost drivers")).toBeLessThan( + labels.indexOf("Technology cost trend"), + ); + }); + }); + + describe("layout", () => { + it("labels each row once for all columns, not per cell", () => { + renderGrid([pathway("a"), pathway("b"), pathway("c")]); + + expect(screen.getAllByText("Policy ambition")).toHaveLength(1); + }); + + it("gives every column a cell in every row", () => { + renderGrid([ + pathway("a", { policies: "A prose." }), + pathway("b"), + pathway("c"), + ]); + + // Two columns lack the driver, so two "not a core driver" notes. + expect(screen.getAllByText(NOT_A_CORE_DRIVER_TEXT)).toHaveLength(2); + }); + + it("sizes the grid to the number of pathways", () => { + const { container } = renderGrid([pathway("a"), pathway("b")]); + const grid = container.firstChild as HTMLElement; + + expect(grid.style.gridTemplateColumns).toBe("repeat(2, 1fr)"); + }); + + it("keeps the group heading bars rather than the detail page's cards", () => { + // The comparison grid deliberately did not adopt the card treatment. + renderGrid([pathway("a"), pathway("b")]); + + const heading = screen.getByRole("heading", { name: "Policies" }); + expect(heading.parentElement).toHaveClass("bg-bluespruce"); + expect(heading.parentElement).not.toHaveClass("bg-white"); + }); + + it("separates rows but does not trail a separator after the last", () => { + const { container } = renderGrid([pathway("a"), pathway("b")]); + + const separators = container.querySelectorAll(".border-b"); + // 11 features across five groups, minus one per group. + expect(separators).toHaveLength(11 - 5); + }); + }); + + describe("key feature values", () => { + it("renders each pathway's own values", () => { + renderGrid([pathway("a"), pathway("b")]); + + // One cell per column, so the widest-scope value appears twice. + expect(screen.getAllByText("High ambition policies")).toHaveLength(2); + }); + + it("explains a value on hover, as the detail page does", () => { + // FeatureItem carries the tooltips, and the comparison grid renders + // FeatureItem — so this comes for free and should stay that way. + renderGrid([pathway("a"), pathway("b")]); + + const pill = screen.getAllByText("Carbon price")[0]; + expect(pill.closest("[tabindex]")).not.toBeNull(); + }); + }); + + describe("degenerate input", () => { + it("renders nothing for an empty comparison", () => { + const { container } = renderGrid([]); + expect(within(container).queryAllByRole("heading")).toHaveLength(0); + }); + + it("tolerates a pathway with no coreDrivers object", () => { + const bare = { + id: "bare", + name: { full: "Bare", short: "BARE" }, + keyFeatures: KEY_FEATURES, + } as unknown as PathwayMetadataType; + + expect(() => renderGrid([bare, pathway("b")])).not.toThrow(); + expect(groupHeadings()).toContain("Policies"); + }); + }); +}); diff --git a/src/components/ComparisonKeyFeatures.tsx b/src/components/ComparisonKeyFeatures.tsx index ea4aac4f..c4c3bf61 100644 --- a/src/components/ComparisonKeyFeatures.tsx +++ b/src/components/ComparisonKeyFeatures.tsx @@ -1,22 +1,93 @@ import React from "react"; import { PathwayMetadataType } from "../types"; import { GROUPS, FeatureItem } from "./KeyFeatures"; +import { CoreDriverItem, coreDriverLabel } from "./CoreDrivers"; +import type { CoreDriverKey } from "./CoreDrivers"; interface ComparisonKeyFeaturesProps { pathways: PathwayMetadataType[]; } +/** + * One row of the grid: a label spanning every column, then one cell per + * pathway. A driver row and a feature row differ only in what fills the cell. + */ +interface Row { + key: string; + label: string; + cell: (pathway: PathwayMetadataType) => React.ReactNode; +} + +/** + * The comparison page's "Assumptions & Trends Overview", as the same six + * themed groups the detail page uses — core drivers and key features + * interleaved by theme rather than split into separate sections. + * + * Drivers come before their group's features, matching the detail page: the + * authored narrative reads first and the enumerated values qualify it. + * + * A driver row is dropped when NO compared pathway describes it. For a single + * pathway "not a core driver for this pathway" is a real statement, which is + * why the detail page renders it; but a row where every column says it carries + * no comparative information, whereas one where some columns describe a driver + * and others do not is the most informative row here. A group left with no + * rows at all is skipped, which is what keeps the driver-only "Other" group + * out until its drivers are authored. + */ const ComparisonKeyFeatures: React.FC = ({ pathways, }) => { const n = pathways.length; + // A group's rows come from the feature config rather than the data, so with + // no pathways every heading would still render above empty columns. The page + // guards this (it needs two pathways to render at all), but the component + // should not depend on that. + if (n === 0) return null; + + const describedBySomeone = (driverKey: CoreDriverKey): boolean => + pathways.some((pathway) => pathway.coreDrivers?.[driverKey]); + + const rowsFor = (group: (typeof GROUPS)[number]): Row[] => [ + ...(group.drivers ?? []) + .filter(describedBySomeone) + .map((driverKey): Row => ({ + key: `driver-${driverKey}`, + label: coreDriverLabel(driverKey), + cell: (pathway) => ( + + ), + })), + ...group.features.map((feature): Row => ({ + key: `feature-${feature.key}`, + label: feature.label, + cell: (pathway) => ( + + ), + })), + ]; + + // Resolved before rendering so a group's visibility and its row separators + // are both decided from the same list. + const groups = GROUPS.map((group) => ({ + label: group.label, + rows: rowsFor(group), + })).filter((group) => group.rows.length > 0); + return (
- {GROUPS.map((group, groupIdx) => ( + {groups.map((group, groupIdx) => ( {/* Group header — spans all pathway columns, styled as a section heading */}
= ({
- {/* One row per feature: label spans all columns, then N content cells side by side */} - {group.features.map((feature, featureIdx) => ( - + {group.rows.map((row, rowIdx) => ( + {/* Single label above all pathway cells */}

- {feature.label} + {row.label}

@@ -47,16 +117,12 @@ const ComparisonKeyFeatures: React.FC = ({ key={pathway.id} className="pb-3 min-w-0" > - + {row.cell(pathway)}
))} - {/* Separator between feature rows, not after the last one */} - {featureIdx < group.features.length - 1 && ( + {/* Separator between rows, not after the last one */} + {rowIdx < group.rows.length - 1 && (
({ vi.mock("./NormalizedStackedAreaChart", () => ({ default: vi.fn(() =>
), })); -vi.mock("../utils/geographyUtils", () => ({ - geographyLabel: (geo: string) => geo, -})); - const mockedMultiLineChart = vi.mocked(MultiLineChart); const mockedStackedAreaChart = vi.mocked(NormalizedStackedAreaChart); @@ -23,6 +19,7 @@ function makeEntry( pathwayId: string, geos: string[], metrics: string[] = ["technologyMix"], + sectorSegment: string[] = ["Power generation"], ): ComparisonPlotsEntry { return { pathwayId, @@ -31,6 +28,7 @@ function makeEntry( metrics.flatMap((metric) => [ { sector: "power", + sectorSegment, metric, geography: geo, year: "2020", @@ -40,6 +38,7 @@ function makeEntry( }, { sector: "power", + sectorSegment, metric, geography: geo, year: "2030", @@ -59,14 +58,16 @@ function makeMetricEntry( pathwayId: string, metric: string, values: number[], + geography = "Global", ): ComparisonPlotsEntry { return { pathwayId, timeseriesdata: { data: values.map((value, i) => ({ sector: "power", + sectorSegment: ["Power generation"], metric, - geography: "Global", + geography, year: String(2020 + i * 10), value, unit: "MtCO2e", @@ -93,41 +94,70 @@ describe("ComparisonPlots", () => { ).toBeInTheDocument(); }); - it("hides the geography selector when all entries share one geography", () => { - const entries = [makeEntry("p1", ["Global"]), makeEntry("p2", ["Global"])]; - render(); - expect(screen.queryByText("Geography")).not.toBeInTheDocument(); - }); - - it("shows the geography selector when entries cover multiple geographies", () => { + it("offers no geography control of its own", () => { + // The scope header above owns that axis; two geography controls on one page + // would contradict each other, and this one listed the raw union of every + // column's timeseries tokens. const entries = [makeEntry("p1", ["Global"]), makeEntry("p2", ["EU"])]; render(); - expect(screen.getByText("Geography")).toBeInTheDocument(); + + expect(screen.getAllByRole("combobox")).toHaveLength(1); + expect(screen.queryByText("Geography")).not.toBeInTheDocument(); }); - it("populates geography selector with the union of geographies across all entries", () => { - const entries = [makeEntry("p1", ["Global"]), makeEntry("p2", ["EU"])]; - render(); - // Plot type is the first combobox; Geography is the second - const geoSelect = screen.getAllByRole("combobox")[1]; - const options = Array.from(geoSelect.querySelectorAll("option")).map( - (o) => o.textContent, + it("resolves each column's own request", async () => { + // Both columns ask for EU; p1 publishes it, p2 does not, so p2 falls back + // and says so rather than rendering empty. + const entries = [ + makeEntry("p1", ["Global", "EU"]), + makeEntry("p2", ["Global"]), + ]; + render( + , ); - expect(options).toContain("Global"); - expect(options).toContain("EU"); + + expect( + await screen.findByText(/showing Global instead/i), + ).toBeInTheDocument(); + // Both columns name what they actually show. + expect(screen.getByText("EU")).toBeInTheDocument(); + expect(screen.getByText("Global")).toBeInTheDocument(); }); - it("shows the per-panel no-data message when a pathway lacks data for the selected geography", async () => { - // p1 has both Global and EU; p2 only has Global + it("lets each column show a different geography", async () => { + // The point of per-column selection: no shared token exists between many + // real publishers, so the columns have to be able to differ. const entries = [ makeEntry("p1", ["Global", "EU"]), makeEntry("p2", ["Global"]), ]; - render(); + render( + , + ); - // Switch to EU — p2 has no EU data - const geoSelect = screen.getAllByRole("combobox")[1]; - await userEvent.setup().selectOptions(geoSelect, "EU"); + // Both columns resolved exactly what they asked for, so neither falls back. + expect(await screen.findByText("EU")).toBeInTheDocument(); + expect(screen.getByText("Global")).toBeInTheDocument(); + expect(screen.queryByText(/showing .* instead/i)).toBeNull(); + }); + + it("shows the per-panel no-data message when a column has nothing to resolve", () => { + const entries = [ + makeEntry("p1", ["Global"]), + { pathwayId: "p2", timeseriesdata: null }, + ]; + render( + , + ); expect( screen.getByText( @@ -136,6 +166,157 @@ describe("ComparisonPlots", () => { ).toBeInTheDocument(); }); + it("scales the shared y-axis to the rows each column actually shows", async () => { + // p1 resolves EU; p2 falls back to Global. Filtering every column on the + // requested token would compute the axis from rows no chart shows and drop + // p2's range entirely. + const entries = [ + makeMetricEntry("p1", "absoluteEmissions", [100, 200], "EU"), + makeMetricEntry("p2", "absoluteEmissions", [10, 500], "Global"), + ]; + render( + , + ); + await userEvent + .setup() + .selectOptions(screen.getAllByRole("combobox")[0], "Absolute Emissions"); + + expect(mockedMultiLineChart.mock.calls.length).toBeGreaterThan(0); + mockedMultiLineChart.mock.calls.forEach(([props]) => { + expect(props.yMin).toBe(10); + expect(props.yMax).toBe(500); + }); + }); + + describe("sector segment badge", () => { + it("captions each column with the part of the sector the metric covers", async () => { + // Matches the detail page's small multiples, which caption each panel + // with its geography and its sector segment. + const entries = [ + makeEntry("p1", ["Global"]), + makeEntry("p2", ["Global"]), + ]; + render( + , + ); + + // One per column, next to that column's geography. + expect(await screen.findAllByText("Power generation")).toHaveLength(2); + }); + + it("reads each column's segments from its own rows", async () => { + // Recorded per row (#915), so two pathways' series of the same metric + // can cover different parts of the sector. + const entries = [ + makeEntry("p1", ["Global"]), + makeEntry( + "p2", + ["Global"], + ["technologyMix"], + ["Power generation", "Energy storage"], + ), + ]; + render( + , + ); + + expect(await screen.findAllByText("Power generation")).toHaveLength(2); + expect(screen.getAllByText("Energy storage")).toHaveLength(1); + }); + + it("carries the segment's definition as a tooltip", async () => { + const entries = [ + makeEntry("p1", ["Global"]), + makeEntry("p2", ["Global"]), + ]; + render( + , + ); + + // Badge renders its tooltip through TextWithTooltip, whose trigger is the + // outer tabIndex span — the text node itself carries no listeners. + const trigger = ( + await screen.findAllByText("Power generation") + )[0].closest("[tabindex]") as HTMLElement; + fireEvent.focus(trigger); + expect(await screen.findByRole("tooltip")).toHaveTextContent( + /generation/i, + ); + }); + + it("omits the badge for a column with nothing to plot", () => { + // No chart, no caption — the same rule the geography badge follows. + const entries = [ + makeEntry("p1", ["Global"]), + { pathwayId: "p2", timeseriesdata: null }, + ]; + render( + , + ); + + expect(screen.getAllByText("Power generation")).toHaveLength(1); + }); + }); + + it("omits the segment badge for a column whose series has one year", () => { + // PlotPanel draws nothing for a single year, so a badge would describe + // an empty chart. + const oneYear = makeEntry("p2", ["Global"]); + oneYear.timeseriesdata!.data = oneYear.timeseriesdata!.data.slice(0, 1); + render( + , + ); + + expect(screen.getAllByText("Power generation")).toHaveLength(1); + }); + + it("explains itself rather than relabelling Power series for another sector", () => { + const entries = [makeEntry("p1", ["Global"]), makeEntry("p2", ["Global"])]; + render( + , + ); + + expect( + screen.getByText( + /cover the Power sector only.*nothing to show for Steel/, + ), + ).toBeInTheDocument(); + expect(screen.queryByTestId("stacked-area-chart")).not.toBeInTheDocument(); + }); + + it("renders the plots for the Power sector", () => { + const entries = [makeEntry("p1", ["Global"]), makeEntry("p2", ["Global"])]; + render( + , + ); + + expect(screen.getAllByTestId("stacked-area-chart")).toHaveLength(2); + }); + it("renders absolute emissions and emissions intensity as line charts", async () => { const entries = [ makeEntry("p1", ["Global"], ["absoluteEmissions", "emissionsIntensity"]), diff --git a/src/components/ComparisonPlots.tsx b/src/components/ComparisonPlots.tsx index 34dc51ba..42471848 100644 --- a/src/components/ComparisonPlots.tsx +++ b/src/components/ComparisonPlots.tsx @@ -1,172 +1,54 @@ import React, { useCallback, useEffect, useMemo, useState } from "react"; import type { PlotType, TimeSeries, HoveredPoint } from "./PlotSelector"; -import NormalizedStackedAreaChart from "./NormalizedStackedAreaChart"; -import MultiLineChart from "./MultiLineChart"; -import { geographyLabel } from "../utils/geographyUtils"; - -const PLOT_OPTIONS: { value: PlotType; label: string }[] = [ - { value: "technologyMix", label: "Technology Mix" }, - { value: "absoluteEmissions", label: "Absolute Emissions" }, - { value: "emissionsIntensity", label: "Emissions Intensity" }, - { value: "capacity", label: "Capacity" }, - { value: "generation", label: "Generation" }, -]; - -// Width/height per panel depending on how many pathways are compared +import { + PlotPanel, + PLOT_OPTIONS, + hasDataForMetric, + hasDataForMetricAndGeo, +} from "./PlotPanel"; +import Badge from "./Badge"; +import RegionMembersTooltip from "./RegionMembersTooltip"; +import type { Geography } from "../types"; +import { + geographyKind, + geographyLabel, + geographyVariant, + normalizeGeography, +} from "../utils/geographyUtils"; +import { + geographyFallbackNote, + resolveGeography, +} from "../utils/geographyFallback"; +import { getSectorDefinition } from "../utils/timeseriesTaxonomy"; +import { plottedSegments } from "../utils/timeseriesSegments"; +import { getSectorSegmentTooltip } from "../utils/tooltipUtils"; +import { useElementWidth } from "../hooks/useElementWidth"; + +/** + * Panel size before the columns have been measured, and the panel height + * throughout: #842 asks for width only, and a constant height is what keeps + * the columns baseline-aligned with each other. + */ const CHART_DIMS: Record = { 2: { width: 400, height: 240 }, 3: { width: 320, height: 240 }, }; -function hasDataForMetric(data: TimeSeries | null, metric: string): boolean { - if (!data?.data) return false; - const filtered = data.data.filter( - (d) => d.sector === "power" && d.metric === metric, - ); - if (filtered.length === 0) return false; - return new Set(filtered.map((d) => d.year)).size > 1; -} - -function hasDataForMetricAndGeo( - data: TimeSeries | null, - metric: string, - geo: string, -): boolean { - if (!data?.data) return false; - const filtered = data.data.filter( - (d) => d.sector === "power" && d.metric === metric && d.geography === geo, - ); - if (filtered.length === 0) return false; - return new Set(filtered.map((d) => d.year)).size > 1; -} - -// ── Single pathway plot panel ──────────────────────────────────────────────── - -interface PlotPanelProps { - timeseriesdata: TimeSeries | null; - datasetId?: string; - plotType: PlotType; - selectedGeography: string; - dims: { width: number; height: number }; - yMin?: number; - yMax?: number; - hoveredPoint?: HoveredPoint | null; - onHoverPoint?: (point: HoveredPoint | null) => void; -} +/** Quantise the measured width so a window drag does not re-run d3 per pixel. */ +const WIDTH_STEP = 20; -const PlotPanel: React.FC = ({ - timeseriesdata, - datasetId, - plotType, - selectedGeography, - dims, - yMin, - yMax, - hoveredPoint, - onHoverPoint, -}) => { - const filteredData = useMemo(() => { - if (!timeseriesdata?.data || !selectedGeography) return { data: [] }; - return { - data: timeseriesdata.data.filter( - (d) => d.geography === selectedGeography, - ), - }; - }, [timeseriesdata, selectedGeography]); - - if ( - !timeseriesdata || - !selectedGeography || - !hasDataForMetricAndGeo(timeseriesdata, plotType, selectedGeography) - ) { - return ( -
- For this pathway, there is currently no data available for the selected - combination of sector, region, and metric. -
- ); - } +/** + * Stable empty default for `requestedGeographies`. An inline `{}` would be a + * new object every render, invalidating the memo below on every pass. + */ +const NO_REQUESTS: Readonly> = Object.freeze({}); - const key = `${datasetId ?? ""}-${plotType}-${selectedGeography}`; - - const renderChart = () => { - switch (plotType) { - case "technologyMix": - return ( - - ); - case "absoluteEmissions": - return ( - - ); - case "emissionsIntensity": - return ( - - ); - case "capacity": - return ( - - ); - case "generation": - return ( - - ); - default: - return null; - } - }; - - return
{renderChart()}
; -}; +/** + * The plots are power-sector only: `PlotPanel` and `MultiLineChart` both + * hardcode `sector="power"`. Naming it once here keeps that assumption visible, + * matching `PlotGrid`. + */ +const PLOT_SECTOR = getSectorDefinition("power"); // ── Shared plot-type selector + N panels ───────────────────────────────────── @@ -174,15 +56,59 @@ export interface ComparisonPlotsEntry { pathwayId: string; timeseriesdata: TimeSeries | null; datasetId?: string; + /** + * The pathway's own geography metadata. Region membership is + * publication-specific, so the fallback for this column has to be resolved + * against this pathway's mapping and no other's. + */ + pathwayGeography?: Geography | null; } interface ComparisonPlotsProps { entries: ComparisonPlotsEntry[]; + /** + * Each column's requested geography, keyed by pathway id and written in that + * publisher's own spelling. + * + * Per column rather than one shared value because tokens are + * publication-specific — a token one publisher declares may not exist in + * another's data at all. A missing or null entry means "no preference", + * resolving to the broadest geography that column has. + */ + requestedGeographies?: Readonly>; + /** + * The shared sector the reader asked for. The plots are Power-only, so any + * other sector gets an explanation rather than Power series relabelled. + */ + requestedSector?: string | null; } -const ComparisonPlots: React.FC = ({ entries }) => { +const ComparisonPlots: React.FC = ({ + entries, + requestedGeographies = NO_REQUESTS, + requestedSector = null, +}) => { const n = entries.length; - const dims = CHART_DIMS[n] ?? CHART_DIMS[3]; + + /* + Size the charts to the column they sit in (#842), rather than to a constant + that only matched one viewport width. + + One observer suffices: every column is `1fr`, so they share a width. The + charts already re-scale on a width change — MultiLineChart and + NormalizedStackedAreaChart both list `width` in their effect deps — so no + remount is needed, and `dims` stays OUT of PlotPanel's key: keying on it + would drop the shared hover state on every resize tick. + */ + const [columnWidth, firstColumnRef] = useElementWidth(WIDTH_STEP); + const fallbackDims = CHART_DIMS[n] ?? CHART_DIMS[3]; + const dims = useMemo( + () => ({ + width: columnWidth ?? fallbackDims.width, + height: fallbackDims.height, + }), + [columnWidth, fallbackDims], + ); // Derive which plot types have data across any pathway (for any geography) const availablePlotOptions = useMemo( @@ -204,29 +130,36 @@ const ComparisonPlots: React.FC = ({ entries }) => { } }, [availablePlotOptions, selectedPlot]); - // Union of all geographies across all compared pathways - const availableGeographies = useMemo(() => { - const geoSet = new Set(); - entries.forEach((e) => { - e.timeseriesdata?.data?.forEach((d) => { - if (d.geography) geoSet.add(d.geography); - }); - }); - return Array.from(geoSet); - }, [entries]); - - const [selectedGeography, setSelectedGeography] = useState(""); + /* + One resolution per column. - useEffect(() => { - if ( - availableGeographies.length > 0 && - !availableGeographies.includes(selectedGeography) - ) { - setSelectedGeography(availableGeographies[0]); - } - }, [availableGeographies, selectedGeography]); + There is no geography control here: the scope header above owns that axis, + one dropdown per column. Two geography controls on one page would + contradict each other, and this one offered the raw union of every column's + timeseries tokens — a list in which most entries were unplottable for most + columns. + */ + const resolutions = useMemo( + () => + entries.map((entry) => { + const available = new Set(); + entry.timeseriesdata?.data?.forEach((d) => { + if (d.geography) available.add(d.geography); + }); + return resolveGeography( + [...available], + requestedGeographies[entry.pathwayId] ?? null, + entry.pathwayGeography ?? null, + ); + }), + [entries, requestedGeographies], + ); - // Shared y-axis bounds across all pathways for the current plot type + geography + // Shared y-axis bounds across all pathways for the current plot type. + // + // Each entry contributes the rows for ITS OWN resolved geography: filtering + // everything on one token would compute the axis from rows no chart shows, + // and leave columns that fell back scaled to someone else's data. const sharedYBounds = useMemo(() => { const isLineChart = selectedPlot === "capacity" || @@ -237,11 +170,13 @@ const ComparisonPlots: React.FC = ({ entries }) => { if (!isLineChart) return undefined; const allValues: number[] = []; - entries.forEach((e) => { + entries.forEach((e, idx) => { + const used = resolutions[idx]?.used; + if (!used) return; e.timeseriesdata?.data ?.filter( (d) => - d.geography === selectedGeography && + d.geography === used && d.sector === "power" && d.metric === selectedPlot, ) @@ -255,7 +190,7 @@ const ComparisonPlots: React.FC = ({ entries }) => { const yMin = selectedPlot === "emissionsIntensity" ? 0 : Math.min(...allValues); return { yMin, yMax }; - }, [entries, selectedPlot, selectedGeography]); + }, [entries, selectedPlot, resolutions]); const [hoveredPoint, setHoveredPoint] = useState(null); @@ -263,12 +198,20 @@ const ComparisonPlots: React.FC = ({ entries }) => { setHoveredPoint(point); }, []); - // Switching plot type or geography remounts every panel; without this, a - // hover captured just before the switch would linger and apply to the - // newly-mounted panels until the next real hover/leave event. + /* + Switching plot type or geography remounts every panel; without this, a + hover captured just before the switch would linger and apply to the + newly-mounted panels until the next real hover/leave event. + + Keyed on what the panels actually RESOLVED to, not on the requested + selection: the request arrives as an object, and depending on its identity + would clear the shared hover on every render that happened to rebuild it, + which is the same failure mode as putting `dims` in PlotPanel's key. + */ + const resolvedKey = resolutions.map((r) => r.used ?? "").join("|"); useEffect(() => { setHoveredPoint(null); - }, [selectedPlot, selectedGeography]); + }, [selectedPlot, resolvedKey]); const hasAnyData = availablePlotOptions.length > 0; @@ -279,65 +222,50 @@ const ComparisonPlots: React.FC = ({ entries }) => { [], ); - const handleGeoChange = useCallback( - (e: React.ChangeEvent) => { - setSelectedGeography(e.target.value); - }, - [], - ); + /* + The charts are Power-only, so a selection for another sector gets an + explanation rather than Power series under its heading. Gated on + PLOT_SECTOR rather than on the sectors present in the data, because the + hardcoding lives in PlotPanel — a dataset carrying Steel rows still would + not make these charts Steel charts. Move this when PlotPanel takes a sector. + */ + if (requestedSector !== null && requestedSector !== PLOT_SECTOR.displayName) { + return ( +

+ {`The benchmark plots cover the ${PLOT_SECTOR.displayName} sector only, so there is nothing to show for ${requestedSector}. Select ${PLOT_SECTOR.displayName} above to see them.`} +

+ ); + } return (
- {/* Shared filters — spans all pathway columns */} + {/* Shared plot-type selector — spans all pathway columns */}
{hasAnyData ? ( -
-
- - -
- {availableGeographies.length > 1 && ( -
- - + {availablePlotOptions.map((opt) => ( + - ))} - -
- )} + {opt.label} + + ))} +
) : (

@@ -346,25 +274,91 @@ const ComparisonPlots: React.FC = ({ entries }) => { )}

- {/* N chart panels — auto-placed one per column */} - {entries.map((entry) => ( -
- -
- ))} + {/* + Which parts of the sector each column's series covers. Read from that + column's own rows (#915): segments are recorded per row, so two + pathways' series of the same metric can differ, and each column says + what it shows, the way the detail page captions its small multiples. + A column whose rows name no segment renders no segment badge. + */} + {entries.map((entry, idx) => { + const resolution = resolutions[idx]; + const used = resolution?.used ?? ""; + const usedKind = geographyKind(used); + const note = resolution ? geographyFallbackNote(resolution) : null; + // Only for a series PlotPanel draws; see PlotGrid. + const segments = hasDataForMetricAndGeo( + entry.timeseriesdata, + selectedPlot, + used, + ) + ? plottedSegments( + entry.timeseriesdata?.data, + PLOT_SECTOR, + selectedPlot, + used, + ) + : []; + + return ( +
+ + + {/* + Which geography this column actually shows. Per column rather + than once for the grid, because the columns can differ — which is + the whole reason the shared request is resolved separately. + */} + {used ? ( +
+ + ) : undefined + } + > + {geographyLabel(normalizeGeography(used))} + + {segments.map((segment) => ( + + {segment} + + ))} +
+ ) : null} + {note ? ( +

{note}

+ ) : null} +
+ ); + })}
); }; diff --git a/src/components/ComparisonRibbon.test.tsx b/src/components/ComparisonRibbon.test.tsx index 3bf00bf8..1c6b20df 100644 --- a/src/components/ComparisonRibbon.test.tsx +++ b/src/components/ComparisonRibbon.test.tsx @@ -1,9 +1,13 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { render, screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import { MemoryRouter } from "react-router"; +import { MemoryRouter, useLocation } from "react-router"; import ComparisonRibbon from "./ComparisonRibbon"; import { useComparison } from "../context/ComparisonContext"; +import { useFilters } from "../context/FilterContext"; +import { EMPTY_FILTERS } from "../context/FilterContext"; +import type { SearchFilters } from "../types"; +import { decodeColumnGeographies } from "../utils/comparisonScope"; // ── Module mocks ──────────────────────────────────────────────────────────── @@ -12,17 +16,37 @@ vi.mock("../context/ComparisonContext", () => ({ MAX_COMPARED: 3, })); +vi.mock("../context/FilterContext", async (importOriginal) => ({ + ...(await importOriginal()), + useFilters: vi.fn(), +})); + vi.mock("../data/pathwayMetadata", () => ({ pathwayMetadata: [ { id: "p1", name: { full: "Pathway One" }, publication: { publisher: { short: "Pub1", full: "Publisher One" } }, + sectors: [{ name: "Power" }, { name: "Steel" }], + geography: { regions: { "South East Asia": ["VN"] }, country: [] }, }, { id: "p2", name: { full: "Pathway Two" }, publication: { publisher: { short: null, full: "Publisher Two" } }, + sectors: [{ name: "Steel" }, { name: "Power" }], + geography: { regions: { "South East Asia": ["VN"] }, country: [] }, + }, + { + // A region label carrying both codec delimiters. + id: "p3", + name: { full: "Pathway Three" }, + publication: { publisher: { short: "Pub3", full: "Publisher Three" } }, + sectors: [{ name: "Power" }], + geography: { + regions: { "Latin America: Central, South": ["BR"] }, + country: [], + }, }, ], })); @@ -40,6 +64,19 @@ const defaultContext = { setRibbonExpanded: vi.fn(), }; +const setFilters = (filters: Partial = {}) => { + vi.mocked(useFilters).mockReturnValue({ + filters: { ...EMPTY_FILTERS, ...filters }, + setFilters: vi.fn(), + resetFilters: vi.fn(), + }); +}; + +/** Prints the query string the ribbon navigated to. */ +const Probe: React.FC = () => ( + {useLocation().search} +); + const renderRibbon = (ids: string[] = [], expanded = false) => { vi.mocked(useComparison).mockReturnValue({ ...defaultContext, @@ -58,6 +95,7 @@ const renderRibbon = (ids: string[] = [], expanded = false) => { describe("ComparisonRibbon", () => { beforeEach(() => { vi.clearAllMocks(); + setFilters(); }); describe("collapsed state (default)", () => { @@ -150,6 +188,33 @@ describe("ComparisonRibbon", () => { ).not.toBeDisabled(); }); + it("counts only ids that still resolve to a pathway", () => { + // A stored id for a pathway that no longer exists renders as an empty + // slot; it used to count towards the minimum of two all the same. + renderRibbon(["p1", "deleted"], true); + expect( + screen.getByRole("button", { name: "Compare (min. 2)" }), + ).toBeDisabled(); + }); + + it("leaves a stale id out of the comparison link", async () => { + vi.mocked(useComparison).mockReturnValue({ + ...defaultContext, + comparedPathwayIds: ["p1", "deleted", "p2"], + ribbonExpanded: true, + }); + render( + + + + , + ); + await userEvent + .setup() + .click(screen.getByRole("button", { name: "Compare" })); + expect(screen.getByTestId("search").textContent).toContain("ids=p1,p2"); + }); + it("is enabled and labelled 'Compare' with 3 pathways selected", () => { renderRibbon(["p1", "p2", "p3"], true); expect( @@ -214,5 +279,82 @@ describe("ComparisonRibbon", () => { expect(clear).toHaveBeenCalledOnce(); }); }); + + describe("compare URL", () => { + const clickCompare = async ( + filters: Partial = {}, + ids = ["p1", "p2"], + ) => { + setFilters(filters); + vi.mocked(useComparison).mockReturnValue({ + ...defaultContext, + comparedPathwayIds: ids, + ribbonExpanded: true, + }); + render( + + + + , + ); + await userEvent + .setup() + .click(screen.getByRole("button", { name: "Compare" })); + const search = screen.getByTestId("search").textContent ?? ""; + const params = new URLSearchParams(search); + // Read back exactly as ComparisonPage does. + const geographies = decodeColumnGeographies(params.get("geography")); + return { search, params, geographies }; + }; + + it("carries the compared ids", async () => { + const { params } = await clickCompare(); + expect(params.get("ids")).toBe("p1,p2"); + }); + + it("seeds the scope so a shared link reproduces what the sender saw", async () => { + // Seeding at navigation time is what keeps the comparison page a pure + // function of its URL. + const { params, geographies } = await clickCompare(); + expect(params.get("sector")).toBe("Power"); + // Geography is per column, keyed by pathway id. + expect(geographies).toEqual({ + p1: "South East Asia", + p2: "South East Asia", + }); + }); + + it("seeds each column in its own publisher's spelling", async () => { + // The whole reason geography is per column: one reader selection maps + // onto a different token for each publication. + const { geographies } = await clickCompare({ + geography: "Southeast Asia", + }); + expect(geographies).toEqual({ + p1: "South East Asia", + p2: "South East Asia", + }); + }); + + it("prefers a search sector the pathways share", async () => { + const { params } = await clickCompare({ sector: "Steel" }); + expect(params.get("sector")).toBe("Steel"); + }); + + it("ignores a search sector the pathways do not share", async () => { + const { params } = await clickCompare({ sector: "Cement" }); + expect(params.get("sector")).toBe("Power"); + }); + + it("round-trips a token containing the codec's delimiters", async () => { + // Geography tokens are publisher prose, not slugs: a comma or colon in + // one must not split it when the comparison page reads the link. + const { geographies } = await clickCompare({}, ["p1", "p3"]); + expect(geographies).toEqual({ + p1: "South East Asia", + p3: "Latin America: Central, South", + }); + }); + }); }); }); diff --git a/src/components/ComparisonRibbon.tsx b/src/components/ComparisonRibbon.tsx index cbd5225e..f84b746d 100644 --- a/src/components/ComparisonRibbon.tsx +++ b/src/components/ComparisonRibbon.tsx @@ -2,7 +2,17 @@ import React from "react"; import { useNavigate } from "react-router"; import { X, Plus, GitCompareArrows, Trash2 } from "lucide-react"; import { useComparison, MAX_COMPARED } from "../context/ComparisonContext"; +import { useFilters } from "../context/FilterContext"; import { pathwayMetadata } from "../data/pathwayMetadata"; +import { + columnGeographyOptions, + defaultGeographyForColumn, + encodeColumnGeographies, + resolveSharedSector, +} from "../utils/comparisonScope"; +import { index } from "../data/index.gen"; +import { pathwayToolAvailability } from "../utils/timeseriesAvailability"; +import type { PathwayMetadataType } from "../types"; const SLOTS = [0, 1, 2] as const; @@ -14,13 +24,64 @@ const ComparisonRibbon: React.FC = () => { ribbonExpanded: expanded, setRibbonExpanded: setExpanded, } = useComparison(); + const { filters } = useFilters(); const navigate = useNavigate(); - const canCompare = comparedPathwayIds.length >= 2; + /* + The pathways the stored ids actually resolve to. An id restored from + sessionStorage can name a pathway that no longer exists; it renders as an + empty slot, so the count, the Compare button and the URL all follow this + list rather than the raw ids -- otherwise the ribbon could show one pathway, + say "2 selected", and send the reader to a comparison of one. + */ + const pathways = React.useMemo( + () => + comparedPathwayIds + .map((id) => pathwayMetadata.find((p) => p.id === id)) + .filter((p): p is PathwayMetadataType => p !== undefined), + [comparedPathwayIds], + ); + const canCompare = pathways.length >= 2; + + /* + Carry the reader's search scope into the comparison URL. + Seeding here rather than on the comparison page keeps that page a pure + function of its URL, and makes a shared link reproduce what the sender + saw — a page seeding from `useFilters()` on mount would resolve an absent + param against the *recipient's* session filters instead. + */ const handleCompare = () => { if (!canCompare) return; - void navigate(`/compare?ids=${comparedPathwayIds.join(",")}`); + + const sector = resolveSharedSector(filters, pathways); + + /* + Geography is one value per column, translated into each publication's own + vocabulary — the reader's "Southeast Asia" becomes ACE's "South East + Asia" and IEA's "Southeast Asia" independently, rather than one of them + winning and the other falling back. + */ + const geographies: Record = {}; + for (const pathway of pathways) { + const options = columnGeographyOptions( + pathway, + pathwayToolAvailability(index.byPathway[pathway.id] ?? []), + ); + const token = defaultGeographyForColumn(filters, pathway, options); + if (token !== null) geographies[pathway.id] = token; + } + + const params = [`ids=${pathways.map((p) => p.id).join(",")}`]; + if (sector !== null) params.push(`sector=${encodeURIComponent(sector)}`); + const encoded = encodeColumnGeographies(geographies); + // Encoded again as a whole: the page reads it back through + // URLSearchParams.get(), which decodes one layer, so without this a comma + // or colon inside a token would come back as a delimiter. The page's own + // setter gets the same second layer from URLSearchParams.set(). + if (encoded !== "") params.push(`geography=${encodeURIComponent(encoded)}`); + + void navigate(`/compare?${params.join("&")}`); }; if (!expanded) { @@ -34,9 +95,9 @@ const ComparisonRibbon: React.FC = () => { Compare Pathways - {comparedPathwayIds.length > 0 && ( + {pathways.length > 0 && ( - {comparedPathwayIds.length} selected + {pathways.length} selected )} diff --git a/src/components/ComparisonScopeHeader.test.tsx b/src/components/ComparisonScopeHeader.test.tsx new file mode 100644 index 00000000..817bd937 --- /dev/null +++ b/src/components/ComparisonScopeHeader.test.tsx @@ -0,0 +1,325 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { render, screen, act, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import ComparisonScopeHeader from "./ComparisonScopeHeader"; +import { MockIntersectionObserver } from "../test/mockIntersectionObserver"; +import { + columnGeographyOptions, + columnsGeographyDivergence, + sharedSectors, + type ColumnGeographyOption, +} from "../utils/comparisonScope"; +import type { PathwayMetadataType } from "../types"; + +/* + The two real shapes this header has to reconcile: IEA, which declares Global + plus named regions with its own "Southeast Asia" spelling, and ACE, which + declares one region spelled with a space and omitting TL. +*/ +const iea = { + id: "IEA-APS-2024", + name: { full: "Announced Pledges Scenario", short: "APS" }, + publication: { publisher: { short: "IEA", full: "IEA Long" } }, + sectors: [{ name: "Power" }, { name: "Steel" }, { name: "Buildings" }], + geography: { + global: true, + regions: { "Southeast Asia": ["ID", "TH", "VN", "TL"] }, + country: ["US"], + }, +} as unknown as PathwayMetadataType; + +const ace = { + id: "ACE-ATS-2024", + name: { full: "ASEAN Target Scenario", short: "ATS" }, + publication: { publisher: { short: "ACE", full: "ACE Long" } }, + sectors: [{ name: "Power" }, { name: "Buildings" }], + geography: { + regions: { "South East Asia": ["ID", "TH", "VN"] }, + country: [], + }, +} as unknown as PathwayMetadataType; + +/** Availability stub: these tokens are plottable, nothing else is. */ +const holding = (...tokens: string[]) => ({ + hasSector: () => true, + hasMetric: () => true, + hasGeography: (raw: string) => tokens.includes(raw), +}); + +const optionsFor = ( + pathways: PathwayMetadataType[], + available: string[] = [], +): Record => + Object.fromEntries( + pathways.map((p) => [ + p.id, + columnGeographyOptions(p, holding(...available)), + ]), + ); + +interface Options { + pathways?: PathwayMetadataType[]; + selectedSector?: string | null; + selectedGeographies?: Record; + available?: string[]; +} + +const renderHeader = ({ + pathways = [iea, ace], + selectedSector = "Power", + selectedGeographies = { + [iea.id]: "Southeast Asia", + [ace.id]: "South East Asia", + }, + available = [], +}: Options = {}) => { + const onSectorChange = vi.fn(); + const onGeographyChange = vi.fn(); + + render( + , + ); + + return { onSectorChange, onGeographyChange, user: userEvent.setup() }; +}; + +/** Drive the sentinel past the top of the viewport. */ +const scrollPast = () => + act(() => { + const observer = MockIntersectionObserver.instances.at(-1); + const sentinel = document.querySelector("[aria-hidden='true']"); + observer?.trigger(sentinel as Element, false, { top: -10 }); + }); + +const columnTrigger = (name: string) => + screen.getByRole("button", { name: `Geography for ${name}` }); + +describe("ComparisonScopeHeader", () => { + beforeEach(() => { + MockIntersectionObserver.instances.length = 0; + }); + + describe("sector axis", () => { + it("offers only the sectors every compared pathway declares", () => { + renderHeader(); + + expect(screen.getByRole("button", { name: "Power" })).toBeInTheDocument(); + expect( + screen.getByRole("button", { name: "Buildings" }), + ).toBeInTheDocument(); + // IEA declares Steel; ACE does not, so it is not a shared axis value. + expect(screen.queryByRole("button", { name: "Steel" })).toBeNull(); + }); + + it("marks the selected sector pressed", () => { + renderHeader(); + + const pressed = screen + .getAllByRole("button") + .filter((b) => b.getAttribute("aria-pressed") === "true"); + expect(pressed.map((b) => b.textContent)).toEqual(["Power"]); + }); + + it("reports a new sector", async () => { + const { onSectorChange, user } = renderHeader(); + await user.click(screen.getByRole("button", { name: "Buildings" })); + + expect(onSectorChange).toHaveBeenCalledWith("Buildings"); + }); + + it("ignores a click on the already-selected sector", async () => { + // The axis always carries a value, so deselection is not reachable. + const { onSectorChange, user } = renderHeader(); + await user.click(screen.getByRole("button", { name: "Power" })); + + expect(onSectorChange).not.toHaveBeenCalled(); + }); + + it("stays one shared row rather than one per column", () => { + // Sector is a closed vocabulary with a shared intersection, unlike + // geography — the asymmetry in this header is deliberate. + renderHeader(); + expect(screen.getAllByRole("button", { name: "Power" })).toHaveLength(1); + }); + }); + + describe("geography, per column", () => { + it("gives every column its own control, named for its pathway", () => { + renderHeader(); + + expect(columnTrigger("APS")).toBeInTheDocument(); + expect(columnTrigger("ATS")).toBeInTheDocument(); + }); + + it("shows each column's own current geography", () => { + renderHeader(); + + expect(columnTrigger("APS")).toHaveTextContent("Southeast Asia"); + expect(columnTrigger("ATS")).toHaveTextContent("South East Asia"); + }); + + it("offers only the geographies that column's publisher declares", async () => { + const { user } = renderHeader(); + await user.click(columnTrigger("ATS")); + + const names = screen + .getAllByRole("option") + .map((o) => o.textContent?.trim()); + expect(names).toEqual(["South East Asia"]); + // IEA's spelling belongs to IEA's column. + expect(names).not.toContain("Southeast Asia"); + }); + + it("labels options plainly, with no publisher suffix", async () => { + // The column carries the provenance, so "(IEA)" would be pure noise — + // and across the loadable pathways it disambiguates nothing anyway. + const { user } = renderHeader(); + await user.click(columnTrigger("APS")); + + const names = screen + .getAllByRole("option") + .map((o) => o.textContent?.trim()); + expect(names).toContain("Southeast Asia"); + expect(names.join(" ")).not.toContain("("); + }); + + it("changes only the column that was used", async () => { + const { onGeographyChange, user } = renderHeader(); + await user.click(columnTrigger("APS")); + await user.click(screen.getByRole("option", { name: "Global" })); + + expect(onGeographyChange).toHaveBeenCalledTimes(1); + expect(onGeographyChange).toHaveBeenCalledWith(iea.id, "Global"); + }); + + it("groups the controls under one axis label", () => { + renderHeader(); + + const group = screen.getByRole("group", { name: "Geography" }); + expect( + within(group).getAllByRole("button", { name: /^Geography for/ }), + ).toHaveLength(2); + }); + + it("degrades to a disabled control for a pathway with no geography", () => { + const bare = { + ...ace, + id: "bare", + name: { full: "Bare", short: "BARE" }, + geography: { regions: {}, country: [] }, + } as unknown as PathwayMetadataType; + + renderHeader({ + pathways: [iea, bare], + selectedGeographies: { [iea.id]: "Global", bare: null }, + }); + + expect(columnTrigger("BARE")).toBeDisabled(); + }); + }); + + describe("availability", () => { + it("shades the geographies this tool cannot plot", async () => { + // 13 of IEA's 15 declared geographies carry no timeseries rows. + const { user } = renderHeader({ available: ["Southeast Asia"] }); + await user.click(columnTrigger("APS")); + + const region = screen.getByRole("option", { name: "Southeast Asia" }); + expect(region.querySelector(".bg-transparent")).toBeNull(); + + const global = screen.getByRole("option", { name: "Global" }); + expect(global.querySelector(".bg-transparent")).not.toBeNull(); + }); + + it("leads with what can be plotted", async () => { + const { user } = renderHeader({ available: ["Southeast Asia"] }); + await user.click(columnTrigger("APS")); + + expect(screen.getAllByRole("option")[0]).toHaveAccessibleName( + "Southeast Asia", + ); + }); + }); + + describe("divergence", () => { + it("says nothing when the columns cover the same countries", () => { + const matching = { + ...iea, + geography: { regions: { "Southeast Asia": ["ID", "TH", "VN"] } }, + } as unknown as PathwayMetadataType; + + renderHeader({ + pathways: [matching, ace], + selectedGeographies: { + [iea.id]: "Southeast Asia", + [ace.id]: "South East Asia", + }, + }); + + expect(screen.queryByText(/not like-for-like/)).toBeNull(); + }); + + it("names the countries only one column includes", () => { + // IEA's South East Asia carries Timor-Leste; ACE's does not. + renderHeader({ + selectedGeographies: { + [iea.id]: "Southeast Asia", + [ace.id]: "South East Asia", + }, + }); + + expect( + screen.getByText(/only APS includes Timor-Leste/), + ).toBeInTheDocument(); + }); + + it("says so when the columns are scoped to different geographies", () => { + renderHeader({ + selectedGeographies: { + [iea.id]: "Global", + [ace.id]: "South East Asia", + }, + }); + + expect( + screen.getByText( + /These columns are not showing the same geography: APS shows Global/, + ), + ).toBeInTheDocument(); + }); + }); + + describe("condensed bar", () => { + it("stays collapsed until the cards scroll away", () => { + renderHeader(); + + const bar = screen.getByText("APS").closest("div[aria-hidden='true']"); + expect(bar).toHaveClass("h-0"); + }); + + it("names each column once the cards scroll away", () => { + renderHeader(); + scrollPast(); + + const bar = screen.getByText("APS").closest("div[aria-hidden='true']"); + expect(bar).toHaveClass("h-8"); + expect(screen.getByText("ATS")).toBeInTheDocument(); + }); + + it("holds no focusables, so nothing hides in a collapsed bar", () => { + renderHeader(); + + const bar = screen.getByText("APS").closest("div[aria-hidden='true']"); + expect(bar?.querySelectorAll("button, a, input")).toHaveLength(0); + }); + }); +}); diff --git a/src/components/ComparisonScopeHeader.tsx b/src/components/ComparisonScopeHeader.tsx new file mode 100644 index 00000000..071d8749 --- /dev/null +++ b/src/components/ComparisonScopeHeader.tsx @@ -0,0 +1,176 @@ +import React, { useId, useState } from "react"; +import type { PathwayMetadataType, Sector } from "../types"; +import BadgeArray from "./BadgeArray"; +import ScopeAxisRow, { LABEL_CLASS, pinSelected } from "./ScopeAxisRow"; +import ColumnGeographySelect from "./ColumnGeographySelect"; +import GeographyDivergenceNotice from "./GeographyDivergenceNotice"; +import { useCondensedOnScroll } from "./PathwayContextRibbon"; +import type { + ColumnGeographyDivergence, + ColumnGeographyOption, +} from "../utils/comparisonScope"; +import { getSectorTooltip } from "../utils/tooltipUtils"; + +interface ComparisonScopeHeaderProps { + /** The compared pathways, in column order. */ + pathways: readonly PathwayMetadataType[]; + /** Sector names every compared pathway declares — one shared axis. */ + sectorOptions: string[]; + selectedSector: string | null; + onSectorChange: (next: string) => void; + /** Each column's own geographies, keyed by pathway id. */ + geographyOptions: Readonly>; + /** Each column's chosen geography, keyed by pathway id. */ + selectedGeographies: Readonly>; + onGeographyChange: (pathwayId: string, next: string) => void; + /** Whether the columns are showing the same thing. */ + divergence: ColumnGeographyDivergence; +} + +const columnLabel = (pathway: PathwayMetadataType): string => + pathway.name.short || pathway.name.full; + +/** + * The comparison page's sticky scope bar: one shared sector axis, one geography + * control per column, and a condensed restatement of the column names that + * appears once the pathway cards scroll away. + * + * The two axes are deliberately shaped differently, because they are different + * kinds of thing: + * + * - **Sector** is a closed vocabulary, so the axis is the intersection of what + * the pathways declare and one shared value scopes every column. It stays a + * badge row. + * - **Geography** is publication-specific. A token resolves to an ISO set only + * against its own publisher's `regions` mapping, and across the loadable + * pathways no token is declared by more than one publisher — so a shared + * value would put every column but one into a fallback. Each column gets its + * own dropdown over its own geographies, mirroring the Geographies coverage + * section further down the page. + * + * The geography row sits on the same column grid as the sections below, so each + * control lines up under the pathway it belongs to. + */ +export const ComparisonScopeHeader: React.FC = ({ + pathways, + sectorOptions, + selectedSector, + onSectorChange, + geographyOptions, + selectedGeographies, + onGeographyChange, + divergence, +}) => { + const [condensed, sentinelRef] = useCondensedOnScroll(); + const [sectorExpanded, setSectorExpanded] = useState(false); + const baseId = useId(); + + const sectors = pinSelected(sectorOptions, selectedSector); + const columns = `repeat(${pathways.length}, minmax(0, 1fr))`; + + return ( + <> + {/* + 1px rather than 0px: a zero-area target's intersectionRatio is + unreliable at threshold 0. The negative margin cancels its own height + so it costs no layout. + */} +