From a582aa32742fe1250a77cd3cba1c7583234fb17b Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:41:42 +0200 Subject: [PATCH 01/25] refactor(scrapers): move the German EV sweep into BaseScraper BNetzA's private sweep becomes BaseScraper.sweepEvRows, with the prefixes it retires taken from a shared list in germany-ev-source.ts. No behaviour change. --- src/scrapers/base.ts | 108 ++++++++++++++++++++++++++++++ src/scrapers/bnetza.test.ts | 12 ++-- src/scrapers/bnetza.ts | 104 ++++------------------------ src/scrapers/germany-ev-source.ts | 10 +++ 4 files changed, 137 insertions(+), 97 deletions(-) diff --git a/src/scrapers/base.ts b/src/scrapers/base.ts index 20614e7..c07d089 100644 --- a/src/scrapers/base.ts +++ b/src/scrapers/base.ts @@ -70,6 +70,21 @@ export interface RawFuelPrice { currency: string; // ISO 4217 } +// A sweep deletes rows not touched within the run plus this margin. +const SWEEP_MARGIN_SECONDS = 300; + +export interface EvSweepOptions { + /** externalId prefix this scraper writes, e.g. "bnetza-". */ + ownPrefix: string; + /** Prefixes of the sources this one replaces in the same country. */ + retirePrefixes: string[]; + /** Fewer rows refreshed by this run than this, and nothing is deleted. */ + minStations: number; + runDurationMs: number; + /** For the log lines, e.g. "the register". */ + label: string; +} + // --------------------------------------------------------------------------- // Per-currency price validation bands // --------------------------------------------------------------------------- @@ -448,4 +463,97 @@ export abstract class BaseScraper { await tx.$executeRawUnsafe(sql, ...params); return batch.length; } + + // --------------------------------------------------------------------------- + // EV sweep + // --------------------------------------------------------------------------- + + /** + * Take out the EV rows nothing else will. `run()` only orphan-cleans + * price-less `fuel` stations, so EV rows accumulate forever: + * + * 1. Own rows this run didn't touch: chargers that left the upstream, or + * whose derived ID churned because coordinates were corrected. + * 2. The rows of the sources this one replaces (`retirePrefixes`). A + * country is served by exactly one EV source at a time, so once this + * one has landed those are pure duplicates. + * + * Both are gated on a DB-counted floor rather than on the `stationsUpserted` + * the pipeline reports, because that figure is `batch.length` regardless of + * what the database did. The count covers only rows this run touched: + * yesterday's 74k rows must not vouch for a degraded fetch that refreshed 42 + * of them. + */ + protected async sweepEvRows(opts: EvSweepOptions): Promise { + const adapter = new PrismaPg({ connectionString: process.env.DATABASE_URL! }); + const prisma = new PrismaClient({ adapter }); + try { + // Cutoff expressed as a duration back from the database clock rather than + // as an app-side timestamp, so clock skew between app and DB cannot make + // this delete rows that were just written. The same cutoff separates + // "refreshed by this run" from "stale", so a row is always on exactly + // one side of it. + const staleSeconds = opts.runDurationMs / 1000 + SWEEP_MARGIN_SECONDS; + const own = `${opts.ownPrefix}%`; + + const counts: Array<{ count: bigint }> = await prisma.$queryRawUnsafe( + `SELECT count(*) FROM stations + WHERE country = $2 AND station_type = 'ev_charger' + AND external_id LIKE $3 + AND updated_at >= NOW() - make_interval(secs => $1::float8)`, + staleSeconds, + this.country, + own, + ); + const refreshed = Number(counts[0]?.count ?? 0); + if (refreshed < opts.minStations) { + console.warn( + `[${this.source}] Only ${refreshed} station(s) refreshed by this run ` + + `(floor ${opts.minStations}) — skipping cleanup`, + ); + return; + } + + const stale: Array<{ count: bigint }> = await prisma.$queryRawUnsafe( + `WITH deleted AS ( + DELETE FROM stations + WHERE country = $2 + AND station_type = 'ev_charger' + AND external_id LIKE $3 + AND updated_at < NOW() - make_interval(secs => $1::float8) + RETURNING id + ) SELECT count(*) FROM deleted`, + staleSeconds, + this.country, + own, + ); + const staleCount = Number(stale[0]?.count ?? 0); + if (staleCount > 0) { + console.log(`[${this.source}] Removed ${staleCount} station(s) no longer in ${opts.label}`); + } + + if (opts.retirePrefixes.length === 0) return; + const retired: Array<{ count: bigint }> = await prisma.$queryRawUnsafe( + `WITH deleted AS ( + DELETE FROM stations + WHERE country = $1 + AND station_type = 'ev_charger' + AND external_id LIKE ANY($2::text[]) + RETURNING id + ) SELECT count(*) FROM deleted`, + this.country, + opts.retirePrefixes.map((p) => `${p}%`), + ); + const retiredCount = Number(retired[0]?.count ?? 0); + if (retiredCount > 0) { + console.log( + `[${this.source}] Retired ${retiredCount} superseded row(s) ` + + `(${opts.retirePrefixes.join(", ")}) for ${this.country} ` + + `(${refreshed} ${opts.label} stations refreshed this run)`, + ); + } + } finally { + await prisma.$disconnect().catch(() => {}); + } + } } diff --git a/src/scrapers/bnetza.test.ts b/src/scrapers/bnetza.test.ts index 46ce817..2ea14e8 100644 --- a/src/scrapers/bnetza.test.ts +++ b/src/scrapers/bnetza.test.ts @@ -484,20 +484,20 @@ describe("BNetzAScraper cleanup", () => { // The floor counts only rows this run refreshed, using the same cutoff as // the stale sweep, so rows left over from an earlier run cannot clear it. expect(stub.queries[0].sql).toMatch(/count\(\*\)/); - expect(stub.queries[0].sql).toMatch(/external_id LIKE 'bnetza-%'/); expect(stub.queries[0].sql).toMatch(/updated_at >= NOW\(\) - make_interval/); - expect(stub.queries[0].params[0]).toBe(30 + 300); + expect(stub.queries[0].params).toEqual([30 + 300, "DE", "bnetza-%"]); // Stale sweep: scoped to this source, and cut off by a duration measured // back from the DB clock so app/DB clock skew cannot delete fresh rows. expect(stub.queries[1].sql).toMatch(/DELETE FROM stations/); - expect(stub.queries[1].sql).toMatch(/external_id LIKE 'bnetza-%'/); expect(stub.queries[1].sql).toMatch(/updated_at < NOW\(\) - make_interval/); - expect(stub.queries[1].params[0]).toBe(30 + 300); // run duration + margin + expect(stub.queries[1].params).toEqual([30 + 300, "DE", "bnetza-%"]); // run duration + margin - // Retirement: OpenChargeMap's German chargers, which nothing else removes. - expect(stub.queries[2].sql).toMatch(/external_id LIKE 'ocm-%'/); + // Retirement: the other German EV sources' chargers, which nothing else + // removes once BNetzA is the chosen source. + expect(stub.queries[2].sql).toMatch(/DELETE FROM stations/); expect(stub.queries[2].sql).toMatch(/station_type = 'ev_charger'/); + expect(stub.queries[2].params).toEqual(["DE", ["ocm-%"]]); }); it("leaves yesterday's rows alone when a degraded fetch refreshed too few of them", async () => { diff --git a/src/scrapers/bnetza.ts b/src/scrapers/bnetza.ts index 0648e14..e4145a3 100644 --- a/src/scrapers/bnetza.ts +++ b/src/scrapers/bnetza.ts @@ -1,7 +1,6 @@ import { z } from "zod"; -import { PrismaPg } from "@prisma/adapter-pg"; -import { PrismaClient } from "../generated/prisma/client"; import { BaseScraper, maxSanePowerKw, type RawFuelPrice, type RawStation, type ScraperResult } from "./base"; +import { otherGermanyEvPrefixes } from "./germany-ev-source"; // --------------------------------------------------------------------------- // BNetzA Ladesäulenregister — official German EV charging point registry @@ -75,10 +74,6 @@ const rawMinStations = Number(process.env.PUMPERLY_BNETZA_MIN_STATIONS ?? "10000 const MIN_STATIONS = Number.isFinite(rawMinStations) && rawMinStations >= 1 ? Math.floor(rawMinStations) : 10_000; -// Slack added to the staleness cutoff so the sweep can never delete a row -// written in the opening moments of the run it belongs to. -const SWEEP_MARGIN_SECONDS = 300; - // Share of merged rows that may disagree on operator before it is worth // flagging. Measured at ~1.2% on the real file, which is colocated chargers run // by different companies, not a merge fault. @@ -368,33 +363,24 @@ export class BNetzAScraper extends BaseScraper { } /** - * Run the normal pipeline, then take out the rows nothing else will. - * - * `base.run()` only orphan-cleans price-less `fuel` stations, so EV rows - * accumulate forever. Two sweeps are needed: - * - * 1. Stations that left the register (or whose derived ID churned because - * an operator corrected its coordinates) — without this the map degrades - * monotonically, one stale pin per correction. - * 2. The OpenChargeMap rows this source replaces. Germany is served by - * exactly one EV source (see germany-ev-source.ts), so once BNetzA has - * landed, the `ocm-` rows are pure duplicates. Unlike Spain's REVE - * handover there is no gradual backfill to wait out: this file arrives - * complete in a single run, so one healthy run is the whole cutover. - * - * Both are gated on a DB-counted floor rather than on the `stationsUpserted` - * the pipeline reports, because that figure is `batch.length` regardless of - * what the database actually did. The count covers only rows this run - * touched: yesterday's 74k rows must not vouch for a degraded fetch that - * refreshed 42 of them, or sweep 1 would delete the other 73,958 and sweep 2 - * would retire OpenChargeMap on the strength of a near-empty file. + * Run the normal pipeline, then sweep (see BaseScraper.sweepEvRows): stale + * `bnetza-` rows, and the other German EV sources' rows this one replaces. + * Unlike Spain's REVE handover there is no gradual backfill to wait out: + * this file arrives complete in a single run, so one healthy run is the + * whole cutover. */ async run(): Promise { const result = await super.run(); if (result.errors.length > 0) return result; try { - await this.sweep(result.durationMs); + await this.sweepEvRows({ + ownPrefix: "bnetza-", + retirePrefixes: otherGermanyEvPrefixes("bnetza-"), + minStations: MIN_STATIONS, + runDurationMs: result.durationMs, + label: "the register", + }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.error(`[${this.source}] Cleanup failed: ${msg}`); @@ -402,68 +388,4 @@ export class BNetzAScraper extends BaseScraper { } return result; } - - private async sweep(runDurationMs: number): Promise { - const adapter = new PrismaPg({ connectionString: process.env.DATABASE_URL! }); - const prisma = new PrismaClient({ adapter }); - try { - // Cutoff expressed as a duration back from the database clock rather than - // as an app-side timestamp, so clock skew between app and DB cannot make - // this delete rows that were just written. The same cutoff separates - // "refreshed by this run" from "stale" below, so a row is always on - // exactly one side of it. - const staleSeconds = runDurationMs / 1000 + SWEEP_MARGIN_SECONDS; - - const counts: Array<{ count: bigint }> = await prisma.$queryRawUnsafe( - `SELECT count(*) FROM stations - WHERE country = 'DE' AND station_type = 'ev_charger' - AND external_id LIKE 'bnetza-%' - AND updated_at >= NOW() - make_interval(secs => $1::float8)`, - staleSeconds, - ); - const refreshed = Number(counts[0]?.count ?? 0); - if (refreshed < MIN_STATIONS) { - console.warn( - `[${this.source}] Only ${refreshed} station(s) refreshed by this run ` + - `(floor ${MIN_STATIONS}) — skipping cleanup`, - ); - return; - } - - const stale: Array<{ count: bigint }> = await prisma.$queryRawUnsafe( - `WITH deleted AS ( - DELETE FROM stations - WHERE country = 'DE' - AND station_type = 'ev_charger' - AND external_id LIKE 'bnetza-%' - AND updated_at < NOW() - make_interval(secs => $1::float8) - RETURNING id - ) SELECT count(*) FROM deleted`, - staleSeconds, - ); - const staleCount = Number(stale[0]?.count ?? 0); - if (staleCount > 0) { - console.log(`[${this.source}] Removed ${staleCount} station(s) no longer in the register`); - } - - const retired: Array<{ count: bigint }> = await prisma.$queryRawUnsafe( - `WITH deleted AS ( - DELETE FROM stations - WHERE country = 'DE' - AND station_type = 'ev_charger' - AND external_id LIKE 'ocm-%' - RETURNING id - ) SELECT count(*) FROM deleted`, - ); - const retiredCount = Number(retired[0]?.count ?? 0); - if (retiredCount > 0) { - console.log( - `[${this.source}] Retired ${retiredCount} superseded OpenChargeMap row(s) for DE ` + - `(${refreshed} registry stations refreshed this run)`, - ); - } - } finally { - await prisma.$disconnect().catch(() => {}); - } - } } diff --git a/src/scrapers/germany-ev-source.ts b/src/scrapers/germany-ev-source.ts index 5ee7dc6..be1343a 100644 --- a/src/scrapers/germany-ev-source.ts +++ b/src/scrapers/germany-ev-source.ts @@ -21,6 +21,16 @@ // fills up with duplicates again. // --------------------------------------------------------------------------- +// The externalId prefix each source writes. A source's sweep retires every +// other prefix in this list, so a new source only needs adding here. +const GERMANY_EV_PREFIXES = ["ocm-", "bnetza-"] as const; +export type GermanyEvPrefix = (typeof GERMANY_EV_PREFIXES)[number]; + +/** The prefixes of the German EV sources that `own` replaces. */ +export function otherGermanyEvPrefixes(own: GermanyEvPrefix): string[] { + return GERMANY_EV_PREFIXES.filter((p) => p !== own); +} + /** * Collapse Germany's two EV scrapers down to whichever one is configured. * From e7ae3a93d27a69bb8065c1993c4848c79eb5ebbb Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:43:01 +0200 Subject: [PATCH 02/25] feat(db): store EV fees and upstream timestamps on prices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New nullable fuel_prices.ev_fees JSONB column. Scrapers can now pass evFees and reportedAt per price; batches without them write the same SQL as before. EV prices are checked per kWh (EUR 0.10–2.00) instead of against the per-litre bands. --- .../migration.sql | 4 + prisma/schema.prisma | 2 + src/scrapers/base.test.ts | 88 ++++++++++++++++++- src/scrapers/base.ts | 41 +++++++-- src/types/station.ts | 28 ++++++ 5 files changed, 156 insertions(+), 7 deletions(-) create mode 100644 prisma/migrations/20261022000000_fuel_price_ev_fees/migration.sql diff --git a/prisma/migrations/20261022000000_fuel_price_ev_fees/migration.sql b/prisma/migrations/20261022000000_fuel_price_ev_fees/migration.sql new file mode 100644 index 0000000..0b69ee4 --- /dev/null +++ b/prisma/migrations/20261022000000_fuel_price_ev_fees/migration.sql @@ -0,0 +1,4 @@ +-- Extra charges that come on top of an EV tariff's €/kWh, filled by the OCPDB +-- scraper, shaped as EvFees (src/types/station.ts), in the price's currency. +-- Nullable and metadata-only, like max_power_kw: fuel rows keep NULL. +ALTER TABLE "fuel_prices" ADD COLUMN IF NOT EXISTS "ev_fees" JSONB; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 32c40b3..4f8a7b9 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -38,6 +38,8 @@ model FuelPrice { currency String @default("EUR") @db.VarChar(3) reportedAt DateTime @default(now()) @map("reported_at") @db.Timestamptz source String @db.VarChar(50) + /// EV only: fees on top of the per-kWh price, shaped as EvFees (src/types/station.ts). + evFees Json? @map("ev_fees") station Station @relation(fields: [stationId], references: [id], onDelete: Cascade) diff --git a/src/scrapers/base.test.ts b/src/scrapers/base.test.ts index 2d0a265..a115049 100644 --- a/src/scrapers/base.test.ts +++ b/src/scrapers/base.test.ts @@ -22,7 +22,15 @@ vi.mock("../generated/prisma/client", () => ({ })); // Concrete test scraper -import { BaseScraper, bandFor, maxSanePowerKw, sanePowerKw, type RawStation, type RawFuelPrice } from "./base"; +import { + BaseScraper, + bandFor, + isPlausibleEvPrice, + maxSanePowerKw, + sanePowerKw, + type RawStation, + type RawFuelPrice, +} from "./base"; describe("bandFor (per-currency price bands)", () => { it("accepts a plausible HUF price (595)", () => { @@ -274,6 +282,74 @@ describe("BaseScraper.run()", () => { expect(result.pricesUpserted).toBe(2); }); + it("validates EV prices per kWh and writes their fees as JSON", async () => { + scraper.mockStations = [{ ...makeStation("ev1"), stationType: "ev_charger" }]; + scraper.mockPrices = [ + { ...makePrice("ev1", "EV", 0.79, "EUR"), evFees: { perMinute: 0.24, afterMin: 30 } }, // valid + makePrice("ev1", "EV", 0.05, "EUR"), // too low (< 0.10), would pass the per-litre alt band + makePrice("ev1", "EV", 2.5, "EUR"), // too high (> 2.00), would pass the EUR fuel band + ]; + + mockQueryRawUnsafe.mockResolvedValueOnce([{ id: "uuid-ev", external_id: "ev1" }]); + mockQueryRawUnsafe.mockResolvedValueOnce([{ count: BigInt(0) }]); + mockExecuteRawUnsafe.mockResolvedValue(undefined); + + const result = await scraper.run(); + expect(result.pricesUpserted).toBe(1); + + const priceInsert = mockExecuteRawUnsafe.mock.calls.find(([sql]) => + String(sql).includes("INSERT INTO fuel_prices"), + ); + expect(priceInsert?.[0]).toContain("ev_fees"); + expect(priceInsert).toContain(JSON.stringify({ perMinute: 0.24, afterMin: 30 })); + }); + + it("leaves ev_fees out of fuel price inserts", async () => { + scraper.mockStations = [makeStation("s1")]; + scraper.mockPrices = [makePrice("s1", "B7", 1.5, "EUR")]; + + mockQueryRawUnsafe.mockResolvedValueOnce([{ id: "uuid-1", external_id: "s1" }]); + mockQueryRawUnsafe.mockResolvedValueOnce([{ count: BigInt(0) }]); + mockExecuteRawUnsafe.mockResolvedValue(undefined); + + await scraper.run(); + const priceInsert = mockExecuteRawUnsafe.mock.calls.find(([sql]) => + String(sql).includes("INSERT INTO fuel_prices"), + ); + expect(priceInsert?.[0]).not.toContain("ev_fees"); + }); + + it("writes an upstream reported_at when the price carries one, NOW() otherwise", async () => { + scraper.mockStations = [{ ...makeStation("ev1"), stationType: "ev_charger" }]; + scraper.mockPrices = [{ ...makePrice("ev1", "EV", 0.79, "EUR"), reportedAt: new Date("2026-10-20T04:00:00Z") }]; + + mockQueryRawUnsafe.mockResolvedValueOnce([{ id: "uuid-ev", external_id: "ev1" }]); + mockQueryRawUnsafe.mockResolvedValueOnce([{ count: BigInt(0) }]); + mockExecuteRawUnsafe.mockResolvedValue(undefined); + + await scraper.run(); + const priceInsert = mockExecuteRawUnsafe.mock.calls.find(([sql]) => + String(sql).includes("INSERT INTO fuel_prices"), + ); + expect(priceInsert?.[0]).toContain("COALESCE($6::timestamptz, NOW())"); + expect(priceInsert).toContain("2026-10-20T04:00:00.000Z"); + }); + + it("keeps NOW() for prices without an upstream timestamp", async () => { + scraper.mockStations = [makeStation("s1")]; + scraper.mockPrices = [makePrice("s1", "B7", 1.5, "EUR")]; + + mockQueryRawUnsafe.mockResolvedValueOnce([{ id: "uuid-1", external_id: "s1" }]); + mockQueryRawUnsafe.mockResolvedValueOnce([{ count: BigInt(0) }]); + mockExecuteRawUnsafe.mockResolvedValue(undefined); + + await scraper.run(); + const priceInsert = mockExecuteRawUnsafe.mock.calls.find(([sql]) => + String(sql).includes("INSERT INTO fuel_prices"), + ); + expect(priceInsert?.[0]).not.toContain("COALESCE"); + }); + it("drops fuel stations with no valid prices (keeps EV chargers)", async () => { const evStation: RawStation = { ...makeStation("ev1"), @@ -431,6 +507,16 @@ describe("BaseScraper.run()", () => { }); }); +describe("isPlausibleEvPrice", () => { + it("accepts EUR between 0.10 and 2.00 per kWh", () => { + expect(isPlausibleEvPrice(0.1, "EUR")).toBe(true); + expect(isPlausibleEvPrice(2, "EUR")).toBe(true); + expect(isPlausibleEvPrice(0.05, "EUR")).toBe(false); + expect(isPlausibleEvPrice(2.5, "EUR")).toBe(false); + expect(isPlausibleEvPrice(0.5, "CHF")).toBe(false); + }); +}); + describe("sanePowerKw", () => { it("rounds plausible values", () => { expect(sanePowerKw(22)).toBe(22); diff --git a/src/scrapers/base.ts b/src/scrapers/base.ts index c07d089..c9ab7f8 100644 --- a/src/scrapers/base.ts +++ b/src/scrapers/base.ts @@ -1,6 +1,6 @@ import { PrismaPg } from "@prisma/adapter-pg"; import { PrismaClient } from "../generated/prisma/client"; -import type { FuelType } from "../types/station"; +import type { EvFees, FuelType } from "../types/station"; /** * Transaction client type — Prisma's interactive transaction callback receives @@ -66,8 +66,17 @@ export interface RawFuelPrice { /** Must match a RawStation.externalId in the same batch */ stationExternalId: string; fuelType: FuelType; - price: number; // per litre in local currency + price: number; // per litre in local currency (per kWh for EV) currency: string; // ISO 4217 + /** EV only: fees on top of the per-kWh price. */ + evFees?: EvFees; + /** When the upstream last vouched for this price. Unset = now. */ + reportedAt?: Date; +} + +/** A charger price per kWh the map can show. Only EUR chargers are priced today. */ +export function isPlausibleEvPrice(price: number, currency: string): boolean { + return currency === "EUR" && price >= 0.1 && price <= 2; } // A sweep deletes rows not touched within the run plus this margin. @@ -192,6 +201,8 @@ export abstract class BaseScraper { const badPriceBefore = prices.length; const validPrices = prices.filter((p) => { if (p.price <= 0) return false; + // Per kWh, not per litre. + if (p.fuelType === "EV") return isPlausibleEvPrice(p.price, p.currency); if (ALT_FUELS.has(p.fuelType)) return p.price >= 0.05 && p.price < 100; if (p.currency === "EUR" || p.currency === "GBP" || p.currency === "CHF") return p.price >= priceMin && p.price <= priceMax; @@ -283,6 +294,8 @@ export abstract class BaseScraper { price: p.price, currency: p.currency, source: this.source, + evFees: p.evFees, + reportedAt: p.reportedAt, }, ]; }); @@ -438,25 +451,41 @@ export abstract class BaseScraper { price: number; currency: string; source: string; + evFees?: EvFees; + reportedAt?: Date; }>, ): Promise { if (batch.length === 0) return 0; - // Each price needs 5 params: stationId, fuelType, price, currency, source + // Like max_power_kw: only batches that carry fees write ev_fees, so fuel + // scrapers keep working on a database without the column. + const withFees = batch.some((p) => p.evFees != null); + // Only batches that carry an upstream timestamp bind one; the rest keep NOW(). + const withReported = batch.some((p) => p.reportedAt != null); + const perRow = 5 + (withFees ? 1 : 0) + (withReported ? 1 : 0); + + // Each price needs 5 params: stationId, fuelType, price, currency, source, + // plus evFees and reportedAt for batches that carry them. const params: unknown[] = []; const valueClauses: string[] = []; for (let i = 0; i < batch.length; i++) { const p = batch[i]; - const offset = i * 5; + const offset = i * perRow; + let n = offset + 5; + const reported = withReported ? `COALESCE($${++n}::timestamptz, NOW())` : "NOW()"; valueClauses.push( - `($${offset + 1}::uuid, $${offset + 2}, $${offset + 3}, $${offset + 4}, NOW(), $${offset + 5})`, + `($${offset + 1}::uuid, $${offset + 2}, $${offset + 3}, $${offset + 4}, ${reported}, $${offset + 5}` + + (withFees ? `, $${++n}::jsonb` : "") + + `)`, ); params.push(p.stationId, p.fuelType, p.price, p.currency, p.source); + if (withReported) params.push(p.reportedAt ? p.reportedAt.toISOString() : null); + if (withFees) params.push(p.evFees ? JSON.stringify(p.evFees) : null); } const sql = ` - INSERT INTO fuel_prices (station_id, fuel_type, price, currency, reported_at, source) + INSERT INTO fuel_prices (station_id, fuel_type, price, currency, reported_at, source${withFees ? ", ev_fees" : ""}) VALUES ${valueClauses.join(",\n")} `; diff --git a/src/types/station.ts b/src/types/station.ts index 7e90c6b..ab2ba84 100644 --- a/src/types/station.ts +++ b/src/types/station.ts @@ -5,6 +5,32 @@ export type StationType = "fuel" | "ev_charger" | "both"; export type { FuelType } from "./fuel"; import type { FuelType } from "./fuel"; +/** + * Charges an EV tariff adds on top of its per-kWh price, in the price's + * currency. `perMinute` starts `afterMin` minutes into the session (0 = from + * the start). Stored as JSONB in fuel_prices.ev_fees. + */ +export interface EvFees { + /** The highest session fee that can apply: what cost estimates use. */ + sessionFee?: number; + /** Set when the session fee depends on the time, day or session. */ + sessionFeeRules?: SessionFeeRule[]; + perMinute?: number; + afterMin?: number; +} + +/** One session fee and when it applies. No fields besides `fee`: always. */ +export interface SessionFeeRule { + fee: number; + /** "HH:MM" local time. */ + from?: string; + to?: string; + /** ISO weekdays, 1 = Monday. */ + days?: number[]; + /** Applies under other conditions too (energy, power, duration...). */ + other?: boolean; +} + export interface Station { id: string; externalId: string; @@ -57,6 +83,8 @@ export interface StationGeoJSON { originalCurrency?: string; /** EV chargers: highest single-connector power in kW (absent when unknown). */ powerKw?: number; + /** EV chargers: fees on top of the per-kWh `price` (absent when there are none). */ + evFees?: EvFees; routeFraction?: number; detourMin?: number; /** Distance from the query point in km (only set by /api/stations/nearest) */ From 7934e003f1771a5acec298310863a862c41dcb2b Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:45:03 +0200 Subject: [PATCH 03/25] feat(api): return EV prices and fees from the station endpoints The EV queries in /api/stations, /api/stations/nearest and /api/route-stations join each charger's latest EV price instead of returning NULL, and pass ev_fees through as evFees. The station list shows its average in /kWh for chargers. --- src/app/api/route-stations/route.test.ts | 21 ++++++++++---- src/app/api/route-stations/route.ts | 20 +++++++++++--- src/app/api/stations/nearest/route.test.ts | 22 ++++++++++++++- src/app/api/stations/nearest/route.ts | 20 +++++++++++--- src/app/api/stations/route.test.ts | 32 +++++++++++++++++++++- src/app/api/stations/route.ts | 20 +++++++++++--- src/components/search/station-results.tsx | 2 +- 7 files changed, 116 insertions(+), 21 deletions(-) diff --git a/src/app/api/route-stations/route.test.ts b/src/app/api/route-stations/route.test.ts index 091f1ea..6e006a3 100644 --- a/src/app/api/route-stations/route.test.ts +++ b/src/app/api/route-stations/route.test.ts @@ -125,11 +125,12 @@ describe("route-stations API", () => { expect(sql).not.toContain("ST_GeomFromText($1, 4326)::geography"); }); - it("queries EV stations without price join", async () => { + it("keeps unpriced EV chargers along the route", async () => { const { prisma } = await import("@/lib/db"); const evRow = { ...mockStationRow, price: null, reported_at: null, max_power_kw: 150 }; const unknownKw = { ...evRow, id: "unknown-kw", max_power_kw: null }; - vi.mocked(prisma.$queryRawUnsafe).mockResolvedValueOnce([evRow, unknownKw]); + const priced = { ...evRow, id: "priced", price: 0.79, currency: "EUR", ev_fees: { perMinute: 0.24, afterMin: 30 } }; + vi.mocked(prisma.$queryRawUnsafe).mockResolvedValueOnce([evRow, unknownKw, priced]); const { POST } = await import("./route"); const response = (await POST(makeRequest({ @@ -139,23 +140,31 @@ describe("route-stations API", () => { }) as any)) as any; expect(response.status).toBe(200); - expect(response.data.features).toHaveLength(2); - // EV query should not include price + expect(response.data.features).toHaveLength(3); + // Unpriced charger rows (price NULL) carry no price expect(response.data.features[0].properties.price).toBeUndefined(); + expect(response.data.features[0].properties).not.toHaveProperty("evFees"); + // Priced chargers carry the €/kWh and the fee note. + expect(response.data.features[2].properties).toMatchObject({ + price: 0.79, + evFees: { perMinute: 0.24, afterMin: 30 }, + }); // Charger power passes through; unknown power is omitted, not null. expect(response.data.features[0].properties.powerKw).toBe(150); expect(response.data.features[1].properties).not.toHaveProperty("powerKw"); expect(vi.mocked(prisma.$queryRawUnsafe)).toHaveBeenCalledTimes(1); const sql = vi.mocked(prisma.$queryRawUnsafe).mock.calls[0][0] as string; - // EV branch: type filter, same CTE-based spatial WHERE, no price JOIN LATERAL. + // EV branch: type filter, same CTE-based spatial WHERE, and an OUTER price + // join so unpriced chargers stay on the map. expect(sql).toContain("WITH route AS (SELECT ST_GeomFromText($1, 4326) AS g)"); expect(sql).toContain("s.station_type IN ('ev_charger', 'both')"); // EV branch binds: WKT=$1, dx=$2, dy=$3, meters=$4 (no fuel param). expect(sql).toContain("s.geom && ST_Expand(r.g::geometry, $2, $3)"); expect(sql).toContain("ST_DWithin(s.geom::geography, r.g::geography, $4)"); expect(sql).toContain("LIMIT 5000"); - expect(sql).not.toContain("JOIN LATERAL"); + expect(sql).toContain("LEFT JOIN LATERAL"); + expect(sql).toContain("fuel_type = 'EV'"); expect(sql).toContain("s.max_power_kw::int AS max_power_kw"); }); diff --git a/src/app/api/route-stations/route.ts b/src/app/api/route-stations/route.ts index d502e8f..3708992 100644 --- a/src/app/api/route-stations/route.ts +++ b/src/app/api/route-stations/route.ts @@ -3,7 +3,7 @@ import { z } from "zod"; import { prisma } from "@/lib/db"; import { rateLimit, clientIp } from "@/lib/rate-limit"; import { fuelTypeEnum } from "@/types/fuel"; -import type { StationsGeoJSONCollection, StationGeoJSON } from "@/types/station"; +import type { EvFees, StationsGeoJSONCollection, StationGeoJSON } from "@/types/station"; const MAX_COORDINATES = 2000; const MAX_RESULTS = 5000; @@ -58,6 +58,8 @@ interface StationRow { distance_m: number; /** EV query only. */ max_power_kw?: number | null; + /** EV query only: fees on top of the per-kWh price. */ + ev_fees?: EvFees | null; } export async function POST(request: NextRequest) { @@ -117,12 +119,21 @@ export async function POST(request: NextRequest) { s.id, s.external_id AS external_id, s.country AS country, s.name, s.brand, s.address, s.city, ST_X(s.geom) AS longitude, ST_Y(s.geom) AS latitude, - NULL::float AS price, 'EUR' AS currency, - NULL::timestamptz AS reported_at, + fp.price::float AS price, COALESCE(fp.currency, 'EUR') AS currency, + fp.reported_at, fp.ev_fees, s.max_power_kw::int AS max_power_kw, ST_LineLocatePoint(r.g::geometry, s.geom)::float AS route_fraction, ST_Distance(s.geom::geography, r.g::geography)::float AS distance_m - FROM stations s, route r + FROM stations s + CROSS JOIN route r + LEFT JOIN LATERAL ( + SELECT price, currency, reported_at, ev_fees + FROM fuel_prices + WHERE station_id = s.id + AND fuel_type = 'EV' + ORDER BY reported_at DESC NULLS LAST + LIMIT 1 + ) fp ON true WHERE s.station_type IN ('ev_charger', 'both') AND s.geom && ST_Expand(r.g::geometry, $2, $3) AND ST_DWithin(s.geom::geography, r.g::geography, $4) @@ -187,6 +198,7 @@ export async function POST(request: NextRequest) { ...(row.price != null ? { price: row.price } : {}), ...(row.reported_at ? { reportedAt: new Date(row.reported_at).toISOString() } : {}), ...(row.max_power_kw != null ? { powerKw: row.max_power_kw } : {}), + ...(row.ev_fees ? { evFees: row.ev_fees } : {}), routeFraction: row.route_fraction, // detourMin is computed later via /api/route-detour (Valhalla-based) }, diff --git a/src/app/api/stations/nearest/route.test.ts b/src/app/api/stations/nearest/route.test.ts index d57a800..78ae0b5 100644 --- a/src/app/api/stations/nearest/route.test.ts +++ b/src/app/api/stations/nearest/route.test.ts @@ -103,7 +103,7 @@ describe("stations/nearest API", () => { expect(callArgs[4]).toBe(10); }); - it("queries EV stations without price join", async () => { + it("keeps unpriced EV chargers in the results", async () => { vi.mocked(prisma.$queryRawUnsafe).mockResolvedValue([{ ...mockRow, price: null, @@ -124,6 +124,26 @@ describe("stations/nearest API", () => { expect(response.data.features[0].properties.powerKw).toBe(150); }); + it("returns an EV charger's price and fees from an outer price join", async () => { + vi.mocked(prisma.$queryRawUnsafe).mockResolvedValue([{ + ...mockRow, + price: 0.79, + currency: "EUR", + max_power_kw: 22, + ev_fees: { sessionFee: 1.5 }, + }]); + + const response = (await GET(makeRequest({ + ...validParams, + fuel: "EV", + }) as any)) as any; + + expect(response.data.features[0].properties).toMatchObject({ price: 0.79, evFees: { sessionFee: 1.5 } }); + const sqlArg = vi.mocked(prisma.$queryRawUnsafe).mock.calls[0][0] as string; + expect(sqlArg).toContain("LEFT JOIN LATERAL"); + expect(sqlArg).toContain("fuel_type = 'EV'"); + }); + it("sets cache-control header", async () => { vi.mocked(prisma.$queryRawUnsafe).mockResolvedValue([]); diff --git a/src/app/api/stations/nearest/route.ts b/src/app/api/stations/nearest/route.ts index c10f610..536820b 100644 --- a/src/app/api/stations/nearest/route.ts +++ b/src/app/api/stations/nearest/route.ts @@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; import { z } from "zod"; import { prisma } from "@/lib/db"; import { fuelTypeEnum } from "@/types/fuel"; -import type { StationsGeoJSONCollection, StationGeoJSON } from "@/types/station"; +import type { EvFees, StationsGeoJSONCollection, StationGeoJSON } from "@/types/station"; const querySchema = z.object({ lat: z @@ -50,6 +50,8 @@ interface NearestStationRow { distance_km: number; /** EV query only. */ max_power_kw?: number | null; + /** EV query only: fees on top of the per-kWh price. */ + ev_fees?: EvFees | null; } export async function GET(request: NextRequest) { @@ -90,12 +92,21 @@ export async function GET(request: NextRequest) { s.city, ST_X(s.geom) AS longitude, ST_Y(s.geom) AS latitude, - NULL::float AS price, - 'EUR' AS currency, - NULL::timestamptz AS reported_at, + fp.price::float AS price, + COALESCE(fp.currency, 'EUR') AS currency, + fp.reported_at, + fp.ev_fees, s.max_power_kw::int AS max_power_kw, (ST_Distance(s.geom, ST_SetSRID(ST_MakePoint($1, $2), 4326)) * 111.32)::float AS distance_km FROM stations s + LEFT JOIN LATERAL ( + SELECT price, currency, reported_at, ev_fees + FROM fuel_prices + WHERE station_id = s.id + AND fuel_type = 'EV' + ORDER BY reported_at DESC NULLS LAST + LIMIT 1 + ) fp ON true WHERE s.station_type IN ('ev_charger', 'both') AND ST_DWithin( s.geom, @@ -169,6 +180,7 @@ export async function GET(request: NextRequest) { ...(row.price != null ? { price: row.price } : {}), ...(row.reported_at ? { reportedAt: new Date(row.reported_at).toISOString() } : {}), ...(row.max_power_kw != null ? { powerKw: row.max_power_kw } : {}), + ...(row.ev_fees ? { evFees: row.ev_fees } : {}), distanceKm: Math.round(row.distance_km * 1000) / 1000, }, })); diff --git a/src/app/api/stations/route.test.ts b/src/app/api/stations/route.test.ts index fe7c531..dafd316 100644 --- a/src/app/api/stations/route.test.ts +++ b/src/app/api/stations/route.test.ts @@ -80,7 +80,7 @@ describe("stations API", () => { expect(response.headers["Cache-Control"]).toBe("public, s-maxage=60, stale-while-revalidate=300"); }); - it("queries EV stations without price join", async () => { + it("keeps unpriced EV chargers on the map", async () => { vi.mocked(prisma.$queryRawUnsafe).mockResolvedValue([{ ...mockRow, price: null, @@ -103,6 +103,36 @@ describe("stations API", () => { expect(response.data.features[0].properties.powerKw).toBe(150); }); + it("returns an EV charger's price and fees from an outer price join", async () => { + vi.mocked(prisma.$queryRawUnsafe).mockResolvedValue([{ + ...mockRow, + price: 0.79, + currency: "EUR", + max_power_kw: 22, + ev_fees: { perMinute: 0.24, afterMin: 120 }, + }, { + ...mockRow, + id: "no-fees", + price: 0.46, + currency: "EUR", + ev_fees: null, + }]); + + const response = (await GET(makeRequest({ + bbox: "9.1,48.7,9.3,48.85", + fuel: "EV", + }) as any)) as any; + + const [priced, noFees] = response.data.features; + expect(priced.properties).toMatchObject({ price: 0.79, currency: "EUR", evFees: { perMinute: 0.24, afterMin: 120 } }); + expect(noFees.properties.price).toBe(0.46); + expect(noFees.properties).not.toHaveProperty("evFees"); + const sqlArg = vi.mocked(prisma.$queryRawUnsafe).mock.calls[0][0] as string; + expect(sqlArg).toContain("LEFT JOIN LATERAL"); + expect(sqlArg).toContain("fuel_type = 'EV'"); + expect(sqlArg).toContain("ev_fees"); + }); + it("returns 400 when bbox is missing", async () => { const response = (await GET(makeRequest({ fuel: "B7", diff --git a/src/app/api/stations/route.ts b/src/app/api/stations/route.ts index 05fb944..7ce9caa 100644 --- a/src/app/api/stations/route.ts +++ b/src/app/api/stations/route.ts @@ -2,7 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; import { z } from "zod"; import { prisma } from "@/lib/db"; import { fuelTypeEnum } from "@/types/fuel"; -import type { StationsGeoJSONCollection, StationGeoJSON } from "@/types/station"; +import type { EvFees, StationsGeoJSONCollection, StationGeoJSON } from "@/types/station"; const RESULT_LIMIT = 20000; @@ -42,6 +42,8 @@ interface StationRow { reported_at: Date | null; /** EV query only. */ max_power_kw?: number | null; + /** EV query only: fees on top of the per-kWh price. */ + ev_fees?: EvFees | null; } export async function GET(request: NextRequest) { @@ -79,11 +81,20 @@ export async function GET(request: NextRequest) { s.city, ST_X(s.geom) AS longitude, ST_Y(s.geom) AS latitude, - NULL::float AS price, - 'EUR' AS currency, - NULL::timestamptz AS reported_at, + fp.price::float AS price, + COALESCE(fp.currency, 'EUR') AS currency, + fp.reported_at, + fp.ev_fees, s.max_power_kw::int AS max_power_kw FROM stations s + LEFT JOIN LATERAL ( + SELECT price, currency, reported_at, ev_fees + FROM fuel_prices + WHERE station_id = s.id + AND fuel_type = 'EV' + ORDER BY reported_at DESC NULLS LAST + LIMIT 1 + ) fp ON true WHERE s.station_type IN ('ev_charger', 'both') AND ST_Within( s.geom, @@ -153,6 +164,7 @@ export async function GET(request: NextRequest) { ...(row.price != null ? { price: row.price } : {}), ...(row.reported_at ? { reportedAt: new Date(row.reported_at).toISOString() } : {}), ...(row.max_power_kw != null ? { powerKw: row.max_power_kw } : {}), + ...(row.ev_fees ? { evFees: row.ev_fees } : {}), }, })); diff --git a/src/components/search/station-results.tsx b/src/components/search/station-results.tsx index 77bb2ac..5bd21b8 100644 --- a/src/components/search/station-results.tsx +++ b/src/components/search/station-results.tsx @@ -59,7 +59,7 @@ export function StationResults({ {avgPrice != null && ( - {t("stations.avg")} {formatPrice(avgPrice)} {currencySymbol}/L + {t("stations.avg")} {formatPrice(avgPrice)} {currencySymbol}{stationList[0]?.properties.fuelType === "EV" ? "/kWh" : "/L"} )} From 6a001837caa6f8a0cee44e347d2dd0f151851fe2 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:47:10 +0200 Subject: [PATCH 04/25] feat(popup): show EV prices and charging fees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Priced chargers show €/kWh, their power, and any per-minute or per-session fees. A session fee that varies lists each option with when it applies. Unpriced chargers say the price isn't published. Adds 4 strings in all 17 locales. --- src/components/map/station-popup.test.tsx | 65 +++++++++++++++++++++- src/components/map/station-popup.tsx | 49 ++++++++++++++-- src/lib/i18n.tsx | 68 +++++++++++++++++++++++ 3 files changed, 175 insertions(+), 7 deletions(-) diff --git a/src/components/map/station-popup.test.tsx b/src/components/map/station-popup.test.tsx index 505614e..724d87e 100644 --- a/src/components/map/station-popup.test.tsx +++ b/src/components/map/station-popup.test.tsx @@ -13,7 +13,7 @@ vi.mock("react-map-gl/maplibre", () => ({ // i18n: echo the key back so we can assert on stable identifiers. vi.mock("@/lib/i18n", () => ({ - useI18n: () => ({ t: (k: string) => k }), + useI18n: () => ({ t: (k: string) => k, locale: "en" }), })); // currency: native display (not converted). Keep the real CURRENCIES table so @@ -118,6 +118,8 @@ describe("StationPopup", () => { expect(screen.getByText("150")).toBeInTheDocument(); expect(screen.getByText("kW")).toBeInTheDocument(); expect(screen.getByText(/popup.maxPower/)).toBeInTheDocument(); + expect(screen.getByText(/popup.priceNotPublished/)).toBeInTheDocument(); + expect(screen.queryByText(/popup.noPrice/)).not.toBeInTheDocument(); }); it("says so when an EV charger's power is unknown", () => { @@ -125,4 +127,65 @@ describe("StationPopup", () => { expect(screen.getByText("popup.powerUnknown")).toBeInTheDocument(); expect(screen.queryByText("kW")).not.toBeInTheDocument(); }); + + it("prices a charger per kWh and shows its power next to the label", () => { + render( {}} />); + expect(screen.getByText("0.790")).toBeInTheDocument(); + expect(screen.getByText("€/kWh")).toBeInTheDocument(); + expect(screen.queryByText("€/L")).not.toBeInTheDocument(); + expect(screen.getByText(/· 22 kW/)).toBeInTheDocument(); + }); + + it("lists a charger's per-minute and session fees", () => { + render( + {}} + />, + ); + // i18n is mocked to echo keys, so the "after {n} min" phrase shows as its key. + expect(screen.getByText("+0.06 €/min popup.feeAfter · +1.50 € popup.perSession")).toBeInTheDocument(); + }); + + it("shows a per-minute fee that applies from the start without an 'after'", () => { + render( + {}} />, + ); + expect(screen.getByText("+0.10 €/min")).toBeInTheDocument(); + }); + + it("lists every option of a session fee that varies", () => { + render( + {}} + />, + ); + expect( + screen.getByText( + "+0.50 € popup.perSession (06:00–22:00, Mon–Fri) · +1.00 € popup.perSession (22:00–06:00) · " + + "+0.80 € popup.perSession (Sat, Sun, popup.feeConditions)", + ), + ).toBeInTheDocument(); + }); + + it("shows no fee line for a charger without fees", () => { + render( {}} />); + expect(screen.queryByText(/\/min|popup.perSession/)).not.toBeInTheDocument(); + }); }); diff --git a/src/components/map/station-popup.tsx b/src/components/map/station-popup.tsx index a998ff1..ff2286d 100644 --- a/src/components/map/station-popup.tsx +++ b/src/components/map/station-popup.tsx @@ -2,7 +2,7 @@ import { useState } from "react"; import { Popup } from "react-map-gl/maplibre"; -import type { StationGeoJSON } from "@/types/station"; +import type { SessionFeeRule, StationGeoJSON } from "@/types/station"; import { FUEL_TYPE_MAP } from "@/types/fuel"; import { useI18n } from "@/lib/i18n"; import { useCurrency, CURRENCIES } from "@/lib/currency"; @@ -25,12 +25,28 @@ function timeAgo(iso: string, t: (key: string) => string): string { return `${t("popup.updated")} ${days}d`; } +/** "06:00–22:00, Mon–Fri" for a session fee that only applies sometimes. */ +function feeRuleText(rule: SessionFeeRule, locale: string, t: (key: string) => string): string { + const parts: string[] = []; + if (rule.from || rule.to) parts.push(`${rule.from ?? "00:00"}–${rule.to ?? "24:00"}`); + if (rule.days?.length) { + // 2024-01-01 was a Monday, so day n of that week is ISO weekday n. + const fmt = new Intl.DateTimeFormat(locale, { weekday: "short", timeZone: "UTC" }); + const name = (d: number) => fmt.format(new Date(Date.UTC(2024, 0, d))); + const d = rule.days; + const run = d.length > 2 && d.every((v, i) => i === 0 || v === d[i - 1] + 1); + parts.push(run ? `${name(d[0])}–${name(d[d.length - 1])}` : d.map(name).join(", ")); + } + if (rule.other) parts.push(t("popup.feeConditions")); + return parts.length > 0 ? ` (${parts.join(", ")})` : ""; +} + function symbolFor(code: string): string { return CURRENCIES.find((c) => c.code === code)?.symbol ?? code; } export function StationPopup({ station, onClose }: StationPopupProps) { - const { t } = useI18n(); + const { t, locale } = useI18n(); const { decimals: userDecimals, rateInfo } = useCurrency(); const [copied, setCopied] = useState(false); const [linkCopied, setLinkCopied] = useState(false); @@ -48,6 +64,22 @@ export function StationPopup({ station, onClose }: StationPopupProps) { const displaySymbol = stationCurrency?.symbol ?? properties.currency; const displayDecimals = isConverted ? userDecimals : (stationCurrency?.decimals ?? 3); + // Chargers are priced per kWh, everything else per litre. + const isEV = properties.fuelType === "EV"; + const unit = isEV ? "/kWh" : "/L"; + // Fees stay in the tariff's own currency: they are not converted. + const feeSymbol = symbolFor(properties.originalCurrency ?? properties.currency); + const fees = properties.evFees; + const feeParts = [ + fees?.perMinute != null && + `+${fees.perMinute.toFixed(2)} ${feeSymbol}/min` + + (fees.afterMin ? ` ${t("popup.feeAfter").replace("{n}", String(fees.afterMin))}` : ""), + // A fee that varies lists every option; the single number is the highest. + ...(fees?.sessionFeeRules ?? (fees?.sessionFee != null ? [{ fee: fees.sessionFee }] : [])).map( + (rule) => `+${rule.fee.toFixed(2)} ${feeSymbol} ${t("popup.perSession")}${feeRuleText(rule, locale, t)}`, + ), + ].filter(Boolean); + const lat = geometry.coordinates[1]; const lng = geometry.coordinates[0]; @@ -121,21 +153,25 @@ export function StationPopup({ station, onClose }: StationPopupProps) { {properties.price.toFixed(displayDecimals)} - {displaySymbol}/L + {displaySymbol}{unit}

{fuelInfo?.label ?? properties.fuelType} + {isEV && properties.powerKw != null && · {properties.powerKw} kW} {properties.reportedAt && ( · {timeAgo(properties.reportedAt, t)} )}

+ {feeParts.length > 0 && ( +

{feeParts.join(" · ")}

+ )} {/* Conversion info */} {isConverted && properties.originalPrice != null && (

- {properties.originalPrice.toFixed(CURRENCIES.find((c) => c.code === properties.originalCurrency)?.decimals ?? 3)} {symbolFor(properties.originalCurrency!)}/L + {properties.originalPrice.toFixed(CURRENCIES.find((c) => c.code === properties.originalCurrency)?.decimals ?? 3)} {symbolFor(properties.originalCurrency!)}{unit} {conversionNote && ( · {conversionNote} )} @@ -143,7 +179,7 @@ export function StationPopup({ station, onClose }: StationPopupProps) { )} ) : properties.fuelType === "EV" ? ( - // Chargers have no prices yet; their power is what tells them apart. + // An unpriced charger: its power is what tells it apart.

{properties.powerKw != null ? (
@@ -156,7 +192,8 @@ export function StationPopup({ station, onClose }: StationPopupProps) {

{t("popup.powerUnknown")}

)}

- {t("popup.maxPower")} · {t("popup.noPrice")} {fuelInfo?.label ?? properties.fuelType} + {/* Chargers the feeds list but no feed prices: say so, so a blank price doesn't read as a bug. */} + {t("popup.maxPower")} · {t("popup.priceNotPublished")}

) : ( diff --git a/src/lib/i18n.tsx b/src/lib/i18n.tsx index 09da697..cc199ba 100644 --- a/src/lib/i18n.tsx +++ b/src/lib/i18n.tsx @@ -114,6 +114,10 @@ export const translations: Record> = { "popup.noPrice": "Sin precio para", "popup.maxPower": "Potencia máxima de carga", "popup.powerUnknown": "Potencia desconocida", + "popup.priceNotPublished": "Precio no publicado", + "popup.feeAfter": "después de {n} min", + "popup.perSession": "por sesión", + "popup.feeConditions": "con condiciones", "popup.navigate": "Cómo llegar", "popup.showOnMap": "Ver en el mapa", "popup.share": "Compartir", @@ -213,6 +217,10 @@ export const translations: Record> = { "popup.noPrice": "No price for", "popup.maxPower": "Max. charging power", "popup.powerUnknown": "Power unknown", + "popup.priceNotPublished": "Price not published", + "popup.feeAfter": "after {n} min", + "popup.perSession": "per session", + "popup.feeConditions": "conditions apply", "popup.navigate": "Get directions", "popup.showOnMap": "Show on map", "popup.share": "Share", @@ -312,6 +320,10 @@ export const translations: Record> = { "popup.noPrice": "Pas de prix pour", "popup.maxPower": "Puissance de charge max.", "popup.powerUnknown": "Puissance inconnue", + "popup.priceNotPublished": "Prix non publié", + "popup.feeAfter": "après {n} min", + "popup.perSession": "par session", + "popup.feeConditions": "sous conditions", "popup.navigate": "Itinéraire", "popup.showOnMap": "Voir sur la carte", "popup.share": "Partager", @@ -411,6 +423,10 @@ export const translations: Record> = { "popup.noPrice": "Kein Preis für", "popup.maxPower": "Max. Ladeleistung", "popup.powerUnknown": "Leistung unbekannt", + "popup.priceNotPublished": "Preis nicht veröffentlicht", + "popup.feeAfter": "nach {n} Min.", + "popup.perSession": "pro Ladevorgang", + "popup.feeConditions": "unter Bedingungen", "popup.navigate": "Route berechnen", "popup.showOnMap": "Auf Karte zeigen", "popup.share": "Teilen", @@ -510,6 +526,10 @@ export const translations: Record> = { "popup.noPrice": "Nessun prezzo per", "popup.maxPower": "Potenza di ricarica max", "popup.powerUnknown": "Potenza sconosciuta", + "popup.priceNotPublished": "Prezzo non pubblicato", + "popup.feeAfter": "dopo {n} min", + "popup.perSession": "per sessione", + "popup.feeConditions": "a condizioni", "popup.navigate": "Indicazioni", "popup.showOnMap": "Mostra sulla mappa", "popup.share": "Condividi", @@ -609,6 +629,10 @@ export const translations: Record> = { "popup.noPrice": "Sem preço para", "popup.maxPower": "Potência máx. de carga", "popup.powerUnknown": "Potência desconhecida", + "popup.priceNotPublished": "Preço não publicado", + "popup.feeAfter": "após {n} min", + "popup.perSession": "por sessão", + "popup.feeConditions": "sob condições", "popup.navigate": "Como chegar", "popup.showOnMap": "Ver no mapa", "popup.share": "Partilhar", @@ -708,6 +732,10 @@ export const translations: Record> = { "popup.noPrice": "Brak ceny dla", "popup.maxPower": "Maks. moc ładowania", "popup.powerUnknown": "Moc nieznana", + "popup.priceNotPublished": "Cena nieopublikowana", + "popup.feeAfter": "po {n} min", + "popup.perSession": "za sesję", + "popup.feeConditions": "z warunkami", "popup.navigate": "Nawiguj", "popup.showOnMap": "Pokaż na mapie", "popup.share": "Udostępnij", @@ -807,6 +835,10 @@ export const translations: Record> = { "popup.noPrice": "Bez ceny pro", "popup.maxPower": "Max. nabíjecí výkon", "popup.powerUnknown": "Výkon neznámý", + "popup.priceNotPublished": "Cena nezveřejněna", + "popup.feeAfter": "po {n} min", + "popup.perSession": "za nabíjení", + "popup.feeConditions": "za podmínek", "popup.navigate": "Navigovat", "popup.showOnMap": "Zobrazit na mapě", "popup.share": "Sdílet", @@ -906,6 +938,10 @@ export const translations: Record> = { "popup.noPrice": "Nincs ár ehhez:", "popup.maxPower": "Max. töltési teljesítmény", "popup.powerUnknown": "Ismeretlen teljesítmény", + "popup.priceNotPublished": "Az ár nincs közzétéve", + "popup.feeAfter": "{n} perc után", + "popup.perSession": "alkalmanként", + "popup.feeConditions": "feltételekkel", "popup.navigate": "Útvonaltervezés", "popup.showOnMap": "Mutasd a térképen", "popup.share": "Megosztás", @@ -1005,6 +1041,10 @@ export const translations: Record> = { "popup.noPrice": "Няма цена за", "popup.maxPower": "Макс. мощност на зареждане", "popup.powerUnknown": "Неизвестна мощност", + "popup.priceNotPublished": "Цената не е публикувана", + "popup.feeAfter": "след {n} мин", + "popup.perSession": "на сесия", + "popup.feeConditions": "при условия", "popup.navigate": "Навигация", "popup.showOnMap": "Покажи на картата", "popup.share": "Сподели", @@ -1104,6 +1144,10 @@ export const translations: Record> = { "popup.noPrice": "Bez ceny pre", "popup.maxPower": "Max. nabíjací výkon", "popup.powerUnknown": "Výkon neznámy", + "popup.priceNotPublished": "Cena nezverejnená", + "popup.feeAfter": "po {n} min", + "popup.perSession": "za nabíjanie", + "popup.feeConditions": "za podmienok", "popup.navigate": "Navigovať", "popup.showOnMap": "Zobraziť na mape", "popup.share": "Zdieľať", @@ -1203,6 +1247,10 @@ export const translations: Record> = { "popup.noPrice": "Ingen pris for", "popup.maxPower": "Maks. ladeeffekt", "popup.powerUnknown": "Ukendt effekt", + "popup.priceNotPublished": "Pris ikke offentliggjort", + "popup.feeAfter": "efter {n} min", + "popup.perSession": "pr. opladning", + "popup.feeConditions": "betingelser gælder", "popup.navigate": "Få rutevejledning", "popup.showOnMap": "Vis på kort", "popup.share": "Del", @@ -1302,6 +1350,10 @@ export const translations: Record> = { "popup.noPrice": "Inget pris för", "popup.maxPower": "Max laddeffekt", "popup.powerUnknown": "Okänd effekt", + "popup.priceNotPublished": "Pris ej publicerat", + "popup.feeAfter": "efter {n} min", + "popup.perSession": "per laddning", + "popup.feeConditions": "villkor gäller", "popup.navigate": "Vägbeskrivning", "popup.showOnMap": "Visa på kartan", "popup.share": "Dela", @@ -1401,6 +1453,10 @@ export const translations: Record> = { "popup.noPrice": "Ingen pris for", "popup.maxPower": "Maks ladeeffekt", "popup.powerUnknown": "Ukjent effekt", + "popup.priceNotPublished": "Pris ikke publisert", + "popup.feeAfter": "etter {n} min", + "popup.perSession": "per lading", + "popup.feeConditions": "vilkår gjelder", "popup.navigate": "Veibeskrivelse", "popup.showOnMap": "Vis på kart", "popup.share": "Del", @@ -1500,6 +1556,10 @@ export const translations: Record> = { "popup.noPrice": "Nema cene za", "popup.maxPower": "Maks. snaga punjenja", "popup.powerUnknown": "Nepoznata snaga", + "popup.priceNotPublished": "Cena nije objavljena", + "popup.feeAfter": "posle {n} min", + "popup.perSession": "po punjenju", + "popup.feeConditions": "uz uslove", "popup.navigate": "Navigacija", "popup.showOnMap": "Prikaži na mapi", "popup.share": "Podeli", @@ -1599,6 +1659,10 @@ export const translations: Record> = { "popup.noPrice": "Ei hintaa:", "popup.maxPower": "Suurin latausteho", "popup.powerUnknown": "Teho tuntematon", + "popup.priceNotPublished": "Hintaa ei ole julkaistu", + "popup.feeAfter": "{n} min jälkeen", + "popup.perSession": "per lataus", + "popup.feeConditions": "ehdoin", "popup.navigate": "Reittiohjeet", "popup.showOnMap": "Näytä kartalla", "popup.share": "Jaa", @@ -1698,6 +1762,10 @@ export const translations: Record> = { "popup.noPrice": "Sense preu per a", "popup.maxPower": "Potència màx. de càrrega", "popup.powerUnknown": "Potència desconeguda", + "popup.priceNotPublished": "Preu no publicat", + "popup.feeAfter": "després de {n} min", + "popup.perSession": "per sessió", + "popup.feeConditions": "amb condicions", "popup.navigate": "Com arribar-hi", "popup.showOnMap": "Mostra-ho en el mapa", "popup.share": "Comparteix", From c4362fac470721a2fade8f8de2cca3d3ceaed6d9 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:49:10 +0200 Subject: [PATCH 05/25] feat(scrapers): add OCPDB as an opt-in German EV source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PUMPERLY_DE_EV_SOURCE=ocpdb pulls Germany's chargers from the Open ChargePoint Database (MobiData BW): the BNetzA register merged with operator feeds, priced with ad-hoc tariffs (gross €/kWh, session and per-minute fees). Tariffs from feeds that failed or went stale are ignored. A run fails rather than wiping the current prices when feed health can't be read. BNetzA stays the default. --- .env.example | 18 +- src/components/nav/legal-modal.tsx | 1 + src/instrumentation-node.ts | 14 +- src/scrapers/bnetza.test.ts | 2 +- src/scrapers/cli.ts | 3 + src/scrapers/germany-ev-source.test.ts | 28 +- src/scrapers/germany-ev-source.ts | 37 +- src/scrapers/ocpdb.test.ts | 637 ++++++++++++++++++++++++ src/scrapers/ocpdb.ts | 648 +++++++++++++++++++++++++ 9 files changed, 1357 insertions(+), 31 deletions(-) create mode 100644 src/scrapers/ocpdb.test.ts create mode 100644 src/scrapers/ocpdb.ts diff --git a/.env.example b/.env.example index d65d082..89b7251 100644 --- a/.env.example +++ b/.env.example @@ -90,14 +90,24 @@ PUMPERLY_DEFAULT_COUNTRY=ES # Set to 0 to disable EV charger scraping (default: enabled when OCM key is set) # PUMPERLY_EV_ENABLED=1 -# Germany EV chargers: which of the two sources supplies them. Defaults to the +# Germany EV chargers: which source supplies them. Defaults to "bnetza", the # BNetzA Ladesäulenregister — the official register every operator of a public # charge point must file into, ~74,000 locations, no API key and no signup. -# Set to "ocm" to keep OpenChargeMap instead. The two never run together: the -# BNetzA scraper retires OpenChargeMap's German rows on its first healthy run, -# and running both would double-pin the country. +# "ocpdb" uses the Open ChargePoint Database via MobiData BW instead: that +# register merged with the operators' own feeds, plus ad-hoc €/kWh prices, also +# keyless. "ocm" keeps OpenChargeMap. Only one ever runs. ocpdb and bnetza +# retire the other sources' German rows on their first healthy run; ocm does +# not, so switching to it leaves the `ocpdb-`/`bnetza-` rows to delete by hand. # PUMPERLY_DE_EV_SOURCE=bnetza +# Stations a run must refresh before the OCPDB scraper is allowed to prune +# stale rows or retire the BNetzA/OpenChargeMap German ones. +# PUMPERLY_OCPDB_MIN_STATIONS=10000 + +# Days an OCPDB operator feed may go without refreshing before its prices are +# ignored (its chargers then show unpriced). Failed feeds are always ignored. +# PUMPERLY_OCPDB_MAX_FEED_AGE_DAYS=3 + # Stations a run must refresh before the BNetzA scraper is allowed to prune # stale rows or retire OpenChargeMap's German ones. A safety floor: a degraded # fetch must not be able to delete a working map. diff --git a/src/components/nav/legal-modal.tsx b/src/components/nav/legal-modal.tsx index 0ba895c..a024055 100644 --- a/src/components/nav/legal-modal.tsx +++ b/src/components/nav/legal-modal.tsx @@ -164,6 +164,7 @@ function SourcesContent() {
  • Open Charge Map — EV charging station locations across all supported countries. Community-maintained, Open Data Commons Open Database License (ODbL). openchargemap.org
  • Mapa REVE — EV charging station locations in Spain. Source: Red Eléctrica de España, S.A.U. Used non-commercially and reproduced without alteration. mapareve.es
  • Ladesäulenregister — EV charging station locations in Germany, from the register operators must file under §5 Ladesäulenverordnung. Source: Bundesnetzagentur.de, Creative Commons Attribution 4.0 (CC BY 4.0). bundesnetzagentur.de
    • +
  • Open ChargePoint Database (OCPDB) — EV charging station locations and ad-hoc prices in Germany, served by MobiData BW (NVBW). Combines the Ladesäulenregister (Bundesnetzagentur.de, CC BY 4.0) with operator feeds published on Mobilithek: EnBW AG (CC BY 4.0); EcoMovement, Chargecloud, Wirelane, e-clearing.net, Tesla and others (CC0). mobidata-bw.de

    • Map and routing

    diff --git a/src/instrumentation-node.ts b/src/instrumentation-node.ts index b54aeb4..995f6d7 100644 --- a/src/instrumentation-node.ts +++ b/src/instrumentation-node.ts @@ -57,6 +57,9 @@ const DEFAULT_INTERVALS: Record = { EV_ES_REVE: 1, // Germany BNetzA Ladesäulenregister — one bulk TSV, regenerated daily. EV_DE_BNETZA: 24, + // Germany OCPDB — register + operator feeds with ad-hoc tariffs; the + // operator feeds refresh several times a day. + EV_DE_OCPDB: 6, }; // Node-only scraper scheduler, loaded from instrumentation.ts. Kept in its own @@ -114,6 +117,7 @@ export async function registerNode() { const { REVEScraper } = await import("./scrapers/reve"); const { resolveSpainEvSource } = await import("./scrapers/spain-ev-source"); const { BNetzAScraper } = await import("./scrapers/bnetza"); + const { OcpdbScraper } = await import("./scrapers/ocpdb"); const { resolveGermanyEvSource } = await import("./scrapers/germany-ev-source"); const { StaticScraper } = await import("./scrapers/static"); const { STATIC_DATASETS } = await import("./scrapers/data"); @@ -205,6 +209,8 @@ export async function registerNode() { EV_ES_REVE: () => new REVEScraper(), // Germany's official EV registry — supersedes EV_DE by default, keyless EV_DE_BNETZA: () => new BNetzAScraper(), + // Germany's register + operator feeds with ad-hoc prices — opt-in, PUMPERLY_DE_EV_SOURCE=ocpdb + EV_DE_OCPDB: () => new OcpdbScraper(), }; // Register community-contributed static datasets (see scrapers/data/README.md). @@ -257,10 +263,12 @@ export async function registerNode() { ); } - // Germany likewise — see scrapers/germany-ev-source.ts, and scrapers/bnetza.ts - // for the sweep that retires the OpenChargeMap rows once BNetzA has landed. + // Germany likewise — see scrapers/germany-ev-source.ts, and the sweeps in + // scrapers/bnetza.ts and scrapers/ocpdb.ts that retire the other sources' rows. countries = resolveGermanyEvSource(countries); - if (countries.includes("EV_DE_BNETZA")) { + if (countries.includes("EV_DE_OCPDB")) { + console.log("[scraper] Germany EV: using OCPDB (BNetzA register + operator feeds, with prices)"); + } else if (countries.includes("EV_DE_BNETZA")) { console.log( "[scraper] Germany EV: using the BNetzA Ladesäulenregister instead of OpenChargeMap", ); diff --git a/src/scrapers/bnetza.test.ts b/src/scrapers/bnetza.test.ts index 2ea14e8..6978884 100644 --- a/src/scrapers/bnetza.test.ts +++ b/src/scrapers/bnetza.test.ts @@ -497,7 +497,7 @@ describe("BNetzAScraper cleanup", () => { // removes once BNetzA is the chosen source. expect(stub.queries[2].sql).toMatch(/DELETE FROM stations/); expect(stub.queries[2].sql).toMatch(/station_type = 'ev_charger'/); - expect(stub.queries[2].params).toEqual(["DE", ["ocm-%"]]); + expect(stub.queries[2].params).toEqual(["DE", ["ocm-%", "ocpdb-%"]]); }); it("leaves yesterday's rows alone when a degraded fetch refreshed too few of them", async () => { diff --git a/src/scrapers/cli.ts b/src/scrapers/cli.ts index 179f27a..9582829 100644 --- a/src/scrapers/cli.ts +++ b/src/scrapers/cli.ts @@ -44,6 +44,7 @@ import { OCMScraper } from "./ocm"; import { REVEScraper } from "./reve"; import { resolveSpainEvSource } from "./spain-ev-source"; import { BNetzAScraper } from "./bnetza"; +import { OcpdbScraper } from "./ocpdb"; import { resolveGermanyEvSource } from "./germany-ev-source"; // --------------------------------------------------------------------------- @@ -145,6 +146,8 @@ const SCRAPERS: Record BaseScraper>> = { EV_ES_REVE: [() => new REVEScraper()], // Germany's official EV registry — supersedes EV_DE by default, keyless EV_DE_BNETZA: [() => new BNetzAScraper()], + // Germany's register + operator feeds with ad-hoc prices — opt-in, PUMPERLY_DE_EV_SOURCE=ocpdb + EV_DE_OCPDB: [() => new OcpdbScraper()], }; function usage(): never { diff --git a/src/scrapers/germany-ev-source.test.ts b/src/scrapers/germany-ev-source.test.ts index 498ab09..d2cfe28 100644 --- a/src/scrapers/germany-ev-source.test.ts +++ b/src/scrapers/germany-ev-source.test.ts @@ -12,18 +12,23 @@ describe("resolveGermanyEvSource", () => { // What the scheduler produces once EV_XX codes are derived. const SCHEDULED = ["DE", "FR", "EV_DE", "EV_FR", "EV_US"]; - // What `--country=all` produces: both German sources are registered. - const ALL = ["DE", "FR", "EV_DE", "EV_FR", "EV_DE_BNETZA", "EV_US"]; + // What `--country=all` produces: every German source is registered. + const ALL = ["DE", "FR", "EV_DE", "EV_FR", "EV_DE_BNETZA", "EV_DE_OCPDB", "EV_US"]; + const GERMAN = ["EV_DE", "EV_DE_BNETZA", "EV_DE_OCPDB"]; + const german = (codes: string[]) => codes.filter((c) => GERMAN.includes(c)); it("uses BNetzA for Germany by default", () => { - expect(resolveGermanyEvSource(SCHEDULED)).toContain("EV_DE_BNETZA"); - expect(resolveGermanyEvSource(SCHEDULED)).not.toContain("EV_DE"); + expect(german(resolveGermanyEvSource(SCHEDULED))).toEqual(["EV_DE_BNETZA"]); + }); + + it("uses OCPDB when PUMPERLY_DE_EV_SOURCE=ocpdb", () => { + vi.stubEnv("PUMPERLY_DE_EV_SOURCE", "ocpdb"); + expect(german(resolveGermanyEvSource(SCHEDULED))).toEqual(["EV_DE_OCPDB"]); }); it("keeps OpenChargeMap when PUMPERLY_DE_EV_SOURCE=ocm", () => { vi.stubEnv("PUMPERLY_DE_EV_SOURCE", "ocm"); - expect(resolveGermanyEvSource(SCHEDULED)).toContain("EV_DE"); - expect(resolveGermanyEvSource(SCHEDULED)).not.toContain("EV_DE_BNETZA"); + expect(german(resolveGermanyEvSource(SCHEDULED))).toEqual(["EV_DE"]); }); it("is case- and whitespace-insensitive about the opt-out", () => { @@ -32,15 +37,16 @@ describe("resolveGermanyEvSource", () => { }); it("falls back to BNetzA when the override is not a source it knows", () => { - vi.stubEnv("PUMPERLY_DE_EV_SOURCE", "nonsense"); - expect(resolveGermanyEvSource(SCHEDULED)).toContain("EV_DE_BNETZA"); + for (const source of ["nonsense", "constructor", "toString"]) { + vi.stubEnv("PUMPERLY_DE_EV_SOURCE", source); + expect(german(resolveGermanyEvSource(SCHEDULED))).toEqual(["EV_DE_BNETZA"]); + } }); it("collapses --country=all down to one German EV source", () => { - for (const source of ["ocm", ""]) { + for (const source of ["ocm", "bnetza", "ocpdb", ""]) { vi.stubEnv("PUMPERLY_DE_EV_SOURCE", source); - const out = resolveGermanyEvSource(ALL); - expect(out.filter((c) => c === "EV_DE" || c === "EV_DE_BNETZA")).toHaveLength(1); + expect(german(resolveGermanyEvSource(ALL))).toHaveLength(1); } }); diff --git a/src/scrapers/germany-ev-source.ts b/src/scrapers/germany-ev-source.ts index be1343a..6cf879f 100644 --- a/src/scrapers/germany-ev-source.ts +++ b/src/scrapers/germany-ev-source.ts @@ -1,18 +1,24 @@ // --------------------------------------------------------------------------- // Which source supplies Germany's EV chargers // --------------------------------------------------------------------------- -// Germany has two: OpenChargeMap (`EV_DE`, crowdsourced, needs an API key) and +// Germany has three: OpenChargeMap (`EV_DE`, crowdsourced, needs an API key), // the BNetzA Ladesäulenregister (`EV_DE_BNETZA`, the official register every -// operator of a public charge point must file into, keyless). +// operator of a public charge point must file into, keyless), and OCPDB +// (`EV_DE_OCPDB`, that register merged with the operators' own feeds, plus +// ad-hoc prices, keyless). // -// They must never both run. The register holds ~74k German locations against +// They must never run together. The register holds ~74k German locations against // OpenChargeMap's crowdsourced subset, and the two overlap heavily — running // both double-pins the country. Worse, once the BNetzA scraper has retired the // `ocm-` rows (see scrapers/bnetza.ts), a single stray OpenChargeMap run puts // every one of them straight back, and nothing garbage-collects EV rows. // // BNetzA wins by default: it is official, it needs no key, and it is the more -// complete of the two. `PUMPERLY_DE_EV_SOURCE=ocm` is the escape hatch. +// complete of the register-based sources. `PUMPERLY_DE_EV_SOURCE=ocpdb` adds +// prices; `=ocm` keeps OpenChargeMap. +// The OCPDB and BNetzA sweeps retire the other sources' rows (see +// scrapers/ocpdb.ts and scrapers/bnetza.ts); OpenChargeMap has no sweep, so +// switching to it leaves the `ocpdb-`/`bnetza-` rows to delete by hand. // // This rule is needed by both the scheduler (instrumentation-node.ts) and the manual // CLI (scrapers/cli.ts), so it lives here rather than in either of them — the @@ -21,9 +27,15 @@ // fills up with duplicates again. // --------------------------------------------------------------------------- +const GERMANY_EV_CODES: Record = { + ocm: "EV_DE", + bnetza: "EV_DE_BNETZA", + ocpdb: "EV_DE_OCPDB", +}; + // The externalId prefix each source writes. A source's sweep retires every // other prefix in this list, so a new source only needs adding here. -const GERMANY_EV_PREFIXES = ["ocm-", "bnetza-"] as const; +const GERMANY_EV_PREFIXES = ["ocm-", "bnetza-", "ocpdb-"] as const; export type GermanyEvPrefix = (typeof GERMANY_EV_PREFIXES)[number]; /** The prefixes of the German EV sources that `own` replaces. */ @@ -32,17 +44,18 @@ export function otherGermanyEvPrefixes(own: GermanyEvPrefix): string[] { } /** - * Collapse Germany's two EV scrapers down to whichever one is configured. + * Collapse Germany's EV scrapers down to whichever one is configured. * * Returns `codes` unchanged when Germany has no EV scraper enabled at all (e.g. - * PUMPERLY_EV_ENABLED=0). Otherwise exactly one of `EV_DE` / `EV_DE_BNETZA` - * survives: OpenChargeMap when PUMPERLY_DE_EV_SOURCE=ocm, BNetzA otherwise. + * PUMPERLY_EV_ENABLED=0). Otherwise exactly one survives: the one named by + * PUMPERLY_DE_EV_SOURCE (ocm | bnetza | ocpdb), BNetzA when unset or unknown. */ export function resolveGermanyEvSource(codes: string[]): string[] { - const hasGermanyEv = codes.includes("EV_DE") || codes.includes("EV_DE_BNETZA"); - const rest = codes.filter((c) => c !== "EV_DE" && c !== "EV_DE_BNETZA"); + const all = Object.values(GERMANY_EV_CODES); + const hasGermanyEv = codes.some((c) => all.includes(c)); + const rest = codes.filter((c) => !all.includes(c)); if (!hasGermanyEv) return rest; - const preferOcm = process.env.PUMPERLY_DE_EV_SOURCE?.trim().toLowerCase() === "ocm"; - rest.push(preferOcm ? "EV_DE" : "EV_DE_BNETZA"); + const chosen = process.env.PUMPERLY_DE_EV_SOURCE?.trim().toLowerCase() ?? ""; + rest.push(Object.hasOwn(GERMANY_EV_CODES, chosen) ? GERMANY_EV_CODES[chosen] : GERMANY_EV_CODES.bnetza); return rest; } diff --git a/src/scrapers/ocpdb.test.ts b/src/scrapers/ocpdb.test.ts new file mode 100644 index 0000000..6cbe4f5 --- /dev/null +++ b/src/scrapers/ocpdb.test.ts @@ -0,0 +1,637 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; + +vi.mock("@prisma/adapter-pg", () => ({ PrismaPg: vi.fn() })); +vi.mock("../generated/prisma/client", () => ({ PrismaClient: vi.fn() })); + +const { grossPrice, summarizeTariff, effectivePerKwh, evseTariffs, liveSources, mergeLocations, stationKey } = + await import("./ocpdb"); +type Tariff = Parameters[0]; +type Location = Parameters[0][number]; + +// Shaped like the live EnBW tariff 132614: net prices, 19% VAT, a per-minute +// blocking fee after 30 minutes. +const ENBW: Tariff = { + id: "132614", + source: "datex2_enbw", + currency: "EUR", + tax_included: "NO", + elements: [ + { + price_components: [ + { type: "ENERGY", price: 0.66386555, taxes: [{ percentage: 19 }], tax_included: false }, + ], + }, + { + price_components: [{ type: "TIME", price: 0.20168067, taxes: [{ percentage: 19 }], tax_included: false }], + restrictions: { min_duration: 1800 }, + }, + ], +}; + +// Shaped like Chargecloud: no tax data at all, a session fee, free TIME steps. +const CHARGECLOUD: Tariff = { + id: "133293", + source: "datex2_chargecloud", + currency: "EUR", + elements: [ + { price_components: [{ type: "FLAT", price: 1.5 }] }, + { price_components: [{ type: "ENERGY", price: 0.46 }] }, + { price_components: [{ type: "TIME", price: 0 }] }, + { price_components: [{ type: "TIME", price: 0.06 }], restrictions: { min_duration: 14400 } }, + ], +}; + +function location(overrides: Partial & { lat?: number; lon?: number; evses?: string[] } = {}): Location { + const { lat = 48.7758, lon = 9.1829, evses = ["e1"], ...rest } = overrides; + return { + source: "bnetza_api", + country: "DEU", + name: null, + address: "Königstraße 1", + postal_code: "70173", + city: "Stuttgart", + state: "Baden-Württemberg", + coordinates: { latitude: lat, longitude: lon }, + operator: { name: "EnBW mobility+ AG und Co.KG" }, + charging_pool: [ + { evses: evses.map((uid) => ({ uid, status: "AVAILABLE", connectors: [{ max_electric_power: 22000 }] })) }, + ], + ...rest, + }; +} + +// ~0.00009° of latitude is 10 m. +const METRES = 1 / 111_320; + +describe("grossPrice", () => { + it("adds VAT to a net component", () => { + expect(grossPrice(ENBW.elements[0].price_components[0], ENBW)).toBe(0.79); + }); + + it("leaves a component that includes tax alone", () => { + expect( + grossPrice({ type: "ENERGY", price: 0.79, taxes: [{ percentage: 19 }], tax_included: true }, { tax_included: "YES" }), + ).toBe(0.79); + }); + + it("reads no tax data as gross", () => { + expect(grossPrice({ type: "ENERGY", price: 0.46 }, { tax_included: null })).toBe(0.46); + }); + + it("falls back to the tariff's flag when the component has none", () => { + expect(grossPrice({ type: "ENERGY", price: 0.5, taxes: [{ percentage: 19 }] }, { tax_included: "NO" })).toBe(0.595); + }); + + it("adds German VAT to a price marked net that gives no rate", () => { + expect(grossPrice({ type: "ENERGY", price: 0.42 }, { tax_included: "NO" })).toBe(0.4998); + expect(grossPrice({ type: "ENERGY", price: 0.42, taxes: [], tax_included: false }, { tax_included: null })).toBe(0.4998); + }); +}); + +describe("summarizeTariff", () => { + it("returns gross €/kWh and the per-minute fee with its start", () => { + expect(summarizeTariff(ENBW)).toEqual({ + energy: 0.79, + currency: "EUR", + fees: { perMinute: 0.24, afterMin: 30 }, + }); + }); + + it("collects the session fee and skips free TIME steps", () => { + expect(summarizeTariff(CHARGECLOUD)).toEqual({ + energy: 0.46, + currency: "EUR", + fees: { sessionFee: 1.5, perMinute: 0.06, afterMin: 240 }, + }); + }); + + it("has no fees for an energy-only tariff", () => { + const t: Tariff = { id: "1", source: "x", currency: "EUR", elements: [{ price_components: [{ type: "ENERGY", price: 0.41 }] }] }; + expect(summarizeTariff(t)?.fees).toEqual({}); + }); + + it("keeps each restricted session fee as a rule and uses the highest", () => { + const t: Tariff = { + id: "1", + source: "x", + currency: "EUR", + elements: [ + { price_components: [{ type: "ENERGY", price: 0.45 }] }, + { price_components: [{ type: "FLAT", price: 0.5 }], restrictions: { start_time: "06:00", end_time: "22:00" } }, + { price_components: [{ type: "FLAT", price: 1 }], restrictions: { start_time: "22:00", end_time: "06:00" } }, + ], + }; + expect(summarizeTariff(t)?.fees).toEqual({ + sessionFee: 1, + sessionFeeRules: [ + { fee: 0.5, from: "06:00", to: "22:00" }, + { fee: 1, from: "22:00", to: "06:00" }, + ], + }); + }); + + it("doesn't list a restricted session fee that repeats the always-on one", () => { + // Shape of live tariff 132762 (datex2_ecomovement): elements repeated under min_duration. + const t: Tariff = { + id: "1", + source: "x", + currency: "EUR", + elements: [ + { price_components: [{ type: "ENERGY", price: 0.5 }], restrictions: { min_duration: 14400 } }, + { price_components: [{ type: "FLAT", price: 0.5 }], restrictions: { min_duration: 14400 } }, + { price_components: [{ type: "ENERGY", price: 0.5 }] }, + { price_components: [{ type: "FLAT", price: 0.5 }] }, + ], + }; + expect(summarizeTariff(t)?.fees).toEqual({ sessionFee: 0.5 }); + }); + + it("lists an always-on session fee next to restricted ones, with weekdays and other conditions", () => { + const t: Tariff = { + id: "1", + source: "x", + currency: "EUR", + elements: [ + { price_components: [{ type: "ENERGY", price: 0.45 }, { type: "FLAT", price: 0.3 }] }, + { price_components: [{ type: "FLAT", price: 0.8 }], restrictions: { day_of_week: ["SATURDAY", "SUNDAY"], min_kwh: 5 } }, + ], + }; + expect(summarizeTariff(t)?.fees).toEqual({ + sessionFee: 0.8, + sessionFeeRules: [{ fee: 0.3 }, { fee: 0.8, days: [6, 7], other: true }], + }); + }); + + it("is null without a per-kWh price", () => { + const t: Tariff = { id: "1", source: "x", currency: "EUR", elements: [{ price_components: [{ type: "FLAT", price: 5 }] }] }; + expect(summarizeTariff(t)).toBeNull(); + }); +}); + +describe("evseTariffs", () => { + it("keeps the cheapest ad-hoc tariff per EVSE and ignores other audiences", () => { + const byEvse = evseTariffs( + [ENBW, CHARGECLOUD], + [ + { tariff_id: ENBW.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "a" }, { evse_uid: "b" }] }, + { tariff_id: CHARGECLOUD.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "b" }] }, + { tariff_id: CHARGECLOUD.id, audience: "REGULAR_PAYMENT", evses: [{ evse_uid: "a" }] }, + ], + ); + expect(byEvse.get("a")?.energy).toBe(0.79); + expect(byEvse.get("b")?.energy).toBe(0.46); + }); + + const energyOnly = (id: string, price: number, flat?: number): Tariff => ({ + id, + source: "x", + currency: "EUR", + elements: [ + { + price_components: [ + { type: "ENERGY", price }, + ...(flat != null ? [{ type: "FLAT", price: flat }] : []), + ], + }, + ], + }); + + it("ignores associations with no audience", () => { + const byEvse = evseTariffs( + [energyOnly("contract", 0.29), energyOnly("adhoc", 0.59)], + [ + { tariff_id: "contract", audience: null, evses: [{ evse_uid: "a" }] }, + { tariff_id: "adhoc", audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "a" }] }, + ], + ); + expect(byEvse.get("a")?.energy).toBe(0.59); + }); + + it("skips an implausible tariff so it can't hide a valid one", () => { + const byEvse = evseTariffs( + [energyOnly("free", 0), energyOnly("promo", 0.05), energyOnly("normal", 0.49)], + ["free", "promo", "normal"].map((id) => ({ tariff_id: id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "a" }] })), + ); + expect(byEvse.get("a")?.energy).toBe(0.49); + }); + + it("ranks by €/kWh with the session fee spread over a session", () => { + const byEvse = evseTariffs( + [energyOnly("cheap-rate", 0.39, 10), energyOnly("no-fee", 0.45)], + ["cheap-rate", "no-fee"].map((id) => ({ tariff_id: id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "a" }] })), + ); + expect(byEvse.get("a")?.energy).toBe(0.45); + expect(effectivePerKwh({ energy: 0.39, currency: "EUR", fees: { sessionFee: 10 } })).toBeCloseTo(0.39 + 10 / 30); + }); + + it("stamps each tariff with its feed's refresh time", () => { + const refreshed = new Date("2026-10-20T04:00:00Z"); + const byEvse = evseTariffs( + [ENBW], + [{ tariff_id: ENBW.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "a" }] }], + new Map([["datex2_enbw", refreshed]]), + ); + expect(byEvse.get("a")?.reportedAt).toEqual(refreshed); + }); +}); + +describe("liveSources", () => { + const now = new Date("2026-10-22T12:00:00Z"); + + it("keeps active feeds refreshed within the window", () => { + const live = liveSources( + [ + { uid: "fresh", static_status: "ACTIVE", static_data_updated_at: "2026-10-22T04:00:00Z" }, + { uid: "old", static_status: "ACTIVE", static_data_updated_at: "2026-09-07T04:00:00Z" }, + { uid: "failed", static_status: "FAILED", static_data_updated_at: "2026-10-22T04:00:00Z" }, + { uid: "never", static_status: "ACTIVE", static_data_updated_at: null }, + ], + now, + 3, + ); + expect([...live]).toEqual([["fresh", new Date("2026-10-22T04:00:00Z")]]); + }); + + it("drops a dead feed's tariffs from the EVSE map", () => { + const byEvse = evseTariffs( + [ENBW, CHARGECLOUD], + [ + { tariff_id: ENBW.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "a" }] }, + { tariff_id: CHARGECLOUD.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "a" }, { evse_uid: "b" }] }, + ], + new Map([["datex2_enbw", new Date()]]), + ); + expect(byEvse.get("a")?.energy).toBe(0.79); + expect(byEvse.has("b")).toBe(false); + }); +}); + +describe("mergeLocations", () => { + const tariffs = evseTariffs( + [ENBW, CHARGECLOUD], + [ + { tariff_id: ENBW.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "enbw" }] }, + { tariff_id: CHARGECLOUD.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "cc" }] }, + ], + ); + + it("builds a priced charger pin", () => { + const { stations, prices } = mergeLocations([location({ evses: ["enbw"] })], tariffs); + expect(stations).toEqual([ + expect.objectContaining({ + externalId: stationKey(48.7758, 9.1829), + brand: "EnBW mobility+ AG und Co.KG", + address: "Königstraße 1, 70173", + city: "Stuttgart", + province: "Baden-Württemberg", + stationType: "ev_charger", + maxPowerKw: 22, + }), + ]); + expect(prices).toEqual([ + { + stationExternalId: stationKey(48.7758, 9.1829), + fuelType: "EV", + price: 0.79, + currency: "EUR", + evFees: { perMinute: 0.24, afterMin: 30 }, + }, + ]); + }); + + it("folds a feed's duplicate into the register pin, keeping register coordinates and the feed's price", () => { + const { stations, prices, stats } = mergeLocations( + [ + // Feed first upstream: the register must still anchor the pin. + location({ source: "datex2_chargecloud", lat: 48.7758 + 10 * METRES, evses: ["cc"] }), + location({ evses: ["unpriced"] }), + ], + tariffs, + ); + expect(stats.merged).toBe(1); + expect(stations).toHaveLength(1); + expect(stations[0].latitude).toBe(48.7758); + expect(prices[0]).toMatchObject({ stationExternalId: stations[0].externalId, price: 0.46 }); + }); + + it("keeps the cheapest price across merged locations, and the highest power", () => { + const fast = location({ source: "datex2_enbw", evses: ["enbw"] }); + fast.charging_pool[0].evses[0].connectors = [{ max_electric_power: 300000 }]; + const { stations, prices } = mergeLocations( + [fast, location({ source: "datex2_chargecloud", lon: 9.1829 + 5 * METRES, evses: ["cc"] })], + tariffs, + ); + expect(stations[0].maxPowerKw).toBe(300); + expect(prices).toHaveLength(1); + expect(prices[0].price).toBe(0.46); + }); + + it("labels a merged pin with the operator whose price it shows", () => { + const { stations, prices } = mergeLocations( + [ + location({ source: "datex2_enbw", name: "Marktplatz", evses: ["enbw"] }), + location({ + source: "datex2_chargecloud", + lon: 9.1829 + 5 * METRES, + operator: { name: "Wirelane GmbH" }, + evses: ["cc"], + }), + ], + tariffs, + ); + expect(prices[0].price).toBe(0.46); + expect(stations[0]).toMatchObject({ brand: "Wirelane GmbH", name: "Wirelane GmbH — Marktplatz" }); + }); + + it("keeps the anchor's operator when the anchor's own price wins", () => { + const { stations, prices } = mergeLocations( + [ + location({ source: "datex2_chargecloud", evses: ["cc"] }), + location({ source: "datex2_wirelane", lon: 9.1829 + 5 * METRES, operator: { name: "Wirelane GmbH" }, evses: ["enbw"] }), + ], + tariffs, + ); + expect(prices[0].price).toBe(0.46); + expect(stations[0].brand).toBe("EnBW mobility+ AG und Co.KG"); + }); + + it("carries the tariff's refresh time onto the price row", () => { + const refreshed = new Date("2026-10-20T04:00:00Z"); + const stamped = evseTariffs( + [ENBW], + [{ tariff_id: ENBW.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "enbw" }] }], + new Map([["datex2_enbw", refreshed]]), + ); + const { prices } = mergeLocations([location({ evses: ["enbw"] })], stamped); + expect(prices[0].reportedAt).toEqual(refreshed); + }); + + it("keeps locations further apart than the merge radius separate", () => { + const { stations } = mergeLocations([location(), location({ lat: 48.7758 + 40 * METRES })], tariffs); + expect(stations).toHaveLength(2); + }); + + it("drops locations outside Germany and those with only removed or planned EVSEs", () => { + const dead = location({ lat: 50 }); + dead.charging_pool[0].evses.forEach((e, i) => (e.status = i % 2 ? "PLANNED" : "REMOVED")); + const { stations, stats } = mergeLocations( + [location({ country: "CHE" }), location({ lat: 40 }), dead], + tariffs, + ); + expect(stations).toEqual([]); + expect(stats.notGermany).toBe(2); + expect(stats.noLiveEvse).toBe(1); + }); + + it("leaves an unpriced pin without a price row", () => { + const { stations, prices } = mergeLocations([location({ evses: ["nothing"] })], tariffs); + expect(stations).toHaveLength(1); + expect(prices).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// fetch(): paging, feed health and bad records, with the network stubbed. +// --------------------------------------------------------------------------- +describe("OcpdbScraper.fetch", () => { + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + const json = (body: unknown) => ({ ok: true, status: 200, json: async () => body }) as unknown as Response; + const page = (items: unknown[], total = items.length) => json({ items, total_count: total }); + + /** Routes each URL to a page; records every URL requested. */ + function stubApi(opts: { sources?: unknown[]; locationsTotal?: number; failOnce?: string; failAlways?: string } = {}) { + const urls: string[] = []; + const failed = new Set(); + vi.stubGlobal( + "fetch", + vi.fn(async (input: string | URL) => { + const url = String(input); + urls.push(url); + if (opts.failAlways && url.includes(opts.failAlways)) { + return { ok: false, status: 503, json: async () => ({}) } as unknown as Response; + } + if (opts.failOnce && url.includes(opts.failOnce) && !failed.has(url)) { + failed.add(url); + throw new Error("socket hang up"); + } + if (url.endsWith("/public/v1/sources")) { + return json({ + items: opts.sources ?? [ + { uid: "datex2_enbw", static_status: "ACTIVE", static_data_updated_at: new Date().toISOString() }, + { uid: "datex2_chargecloud", static_status: "ACTIVE", static_data_updated_at: new Date().toISOString() }, + ], + }); + } + if (url.includes("/tariffs?")) return page([ENBW, CHARGECLOUD]); + if (url.includes("/tariff-associations?")) { + return page([ + { tariff_id: ENBW.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "enbw" }] }, + { tariff_id: CHARGECLOUD.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "cc" }] }, + ]); + } + const offset = Number(new URL(url).searchParams.get("offset")); + // One location per page, a degree of latitude apart so none merge. + // The first page also carries one record the schema rejects. + const items: unknown[] = [location({ lat: 48 + offset / 1000, evses: [offset === 0 ? "enbw" : "cc"] })]; + if (offset === 0) items.push({ source: "broken" }); + return page(items, opts.locationsTotal ?? 1); + }), + ); + return urls; + } + + it("reads every page of locations in steps of 1000", async () => { + const urls = stubApi({ locationsTotal: 2500 }); + const { OcpdbScraper } = await import("./ocpdb"); + + const { stations } = await new OcpdbScraper().fetch(); + + const offsets = urls.filter((u) => u.includes("/locations?")).map((u) => new URL(u).searchParams.get("offset")); + expect(offsets.sort()).toEqual(["0", "1000", "2000"]); + expect(urls.filter((u) => u.includes("/locations?")).every((u) => u.includes("country=DEU&limit=1000"))).toBe(true); + expect(stations).toHaveLength(3); + }); + + it("skips records that fail to parse instead of failing the run", async () => { + stubApi(); + const { OcpdbScraper } = await import("./ocpdb"); + vi.spyOn(console, "log").mockImplementation(() => {}); + + const { stations, prices } = await new OcpdbScraper().fetch(); + + expect(stations).toHaveLength(1); + expect(prices).toEqual([expect.objectContaining({ fuelType: "EV", price: 0.79 })]); + expect(vi.mocked(console.log).mock.calls.flat().join(" ")).toMatch(/1 unparseable/); + }); + + it("leaves a dead feed's chargers on the map without a price", async () => { + stubApi({ + locationsTotal: 2, + sources: [ + { uid: "datex2_enbw", static_status: "FAILED", static_data_updated_at: new Date().toISOString() }, + { uid: "datex2_chargecloud", static_status: "ACTIVE", static_data_updated_at: new Date().toISOString() }, + ], + }); + const { OcpdbScraper } = await import("./ocpdb"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + const { stations, prices } = await new OcpdbScraper().fetch(); + + expect(stations).toHaveLength(1); // offset 0 only: total 2 fits one page + expect(prices).toEqual([]); // that pin's EVSE is priced only by EnBW + expect(warn.mock.calls.flat().join(" ")).toMatch(/datex2_enbw/); + }); + + it("fails the run when feed health is unavailable, so the current prices stay", async () => { + const urls = stubApi({ failAlways: "/public/v1/sources" }); + const { OcpdbScraper } = await import("./ocpdb"); + vi.spyOn(console, "log").mockImplementation(() => {}); + + await expect(new OcpdbScraper().fetch()).rejects.toThrow(/for sources/); + expect(urls.filter((u) => u.includes("/public/v1/sources"))).toHaveLength(2); // one retry + }); + + it("retries feed health once", async () => { + stubApi({ failOnce: "/public/v1/sources" }); + const { OcpdbScraper } = await import("./ocpdb"); + vi.spyOn(console, "log").mockImplementation(() => {}); + + const { stations } = await new OcpdbScraper().fetch(); + expect(stations).toHaveLength(1); + }); + + it("retries a page once", async () => { + const urls = stubApi({ failOnce: "/tariffs?" }); + const { OcpdbScraper } = await import("./ocpdb"); + + await expect(new OcpdbScraper().fetch()).resolves.toBeDefined(); + expect(urls.filter((u) => u.includes("/tariffs?"))).toHaveLength(2); + }); + + it("fails the fetch when a page keeps failing, rather than writing a partial map", async () => { + stubApi({ failAlways: "/locations?" }); + const { OcpdbScraper } = await import("./ocpdb"); + + await expect(new OcpdbScraper().fetch()).rejects.toThrow(/503/); + }); +}); + +// --------------------------------------------------------------------------- +// The cleanup sweep, the only destructive thing this scraper does. base.run() +// is stubbed out so these exercise the gating and the SQL alone. +// --------------------------------------------------------------------------- +describe("OcpdbScraper cleanup", () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.resetModules(); + vi.unstubAllEnvs(); + }); + + async function setup(opts: { results?: Array>; errors?: string[] } = {}) { + const results = [...(opts.results ?? [])]; + const queries: Array<{ sql: string; params: unknown[] }> = []; + const client = { + $queryRawUnsafe: vi.fn(async (sql: string, ...params: unknown[]) => { + queries.push({ sql, params }); + return results.shift() ?? []; + }), + $disconnect: vi.fn(async () => {}), + }; + const { PrismaClient } = await import("../generated/prisma/client"); + const { BaseScraper } = await import("./base"); + // A plain function, not an arrow: this stands in for a constructor. + vi.mocked(PrismaClient).mockImplementation(function () { + return client; + } as never); + vi.spyOn(BaseScraper.prototype, "run").mockResolvedValue({ + country: "DE", + source: "ocpdb", + stationsUpserted: 69160, + pricesUpserted: 25500, + durationMs: 60_000, + errors: opts.errors ?? [], + }); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + const { OcpdbScraper } = await import("./ocpdb"); + return { queries, client, scraper: new OcpdbScraper() }; + } + + it("prunes stale OCPDB rows and retires BNetzA and OpenChargeMap once the floor is cleared", async () => { + const { queries, scraper } = await setup({ + results: [[{ count: BigInt(69160) }], [{ count: BigInt(7) }], [{ count: BigInt(74223) }]], + }); + + const result = await scraper.run(); + + expect(result.errors).toEqual([]); + expect(queries).toHaveLength(3); + + // The floor counts only rows this run refreshed. + expect(queries[0].sql).toMatch(/count\(\*\)/); + expect(queries[0].sql).not.toMatch(/DELETE/); + expect(queries[0].sql).toMatch(/updated_at >= NOW\(\) - make_interval/); + expect(queries[0].params).toEqual([60 + 300, "DE", "ocpdb-%"]); + + // Stale sweep: this source only, cut off by run duration + margin. + expect(queries[1].sql).toMatch(/DELETE FROM stations/); + expect(queries[1].sql).toMatch(/updated_at < NOW\(\) - make_interval/); + expect(queries[1].params).toEqual([60 + 300, "DE", "ocpdb-%"]); + + // Retirement: German chargers only, from the two sources OCPDB replaces. + expect(queries[2].sql).toMatch(/DELETE FROM stations/); + expect(queries[2].sql).toMatch(/station_type = 'ev_charger'/); + expect(queries[2].params).toEqual(["DE", ["ocm-%", "bnetza-%"]]); + }); + + it("deletes nothing when too few stations were refreshed", async () => { + const { queries, scraper } = await setup({ results: [[{ count: BigInt(42) }]] }); + + await scraper.run(); + + expect(queries).toHaveLength(1); + expect(queries[0].sql).not.toMatch(/DELETE/); + }); + + it("does not sweep after a failed pipeline run", async () => { + const { queries, scraper } = await setup({ errors: ["upsert failed"] }); + + const result = await scraper.run(); + + expect(queries).toEqual([]); + expect(result.errors).toEqual(["upsert failed"]); + }); + + it("reports a failed sweep as an error and still disconnects", async () => { + const { client, scraper } = await setup(); + client.$queryRawUnsafe.mockRejectedValueOnce(new Error("connection reset")); + vi.spyOn(console, "error").mockImplementation(() => {}); + + const result = await scraper.run(); + + expect(result.errors).toEqual(["Cleanup: connection reset"]); + expect(client.$disconnect).toHaveBeenCalled(); + }); + + it("honours PUMPERLY_OCPDB_MIN_STATIONS", async () => { + vi.stubEnv("PUMPERLY_OCPDB_MIN_STATIONS", "40"); + const { queries, scraper } = await setup({ results: [[{ count: BigInt(42) }]] }); + + await scraper.run(); + + expect(queries).toHaveLength(3); // 42 now clears the lowered floor + }); + + it("ignores a nonsensical floor rather than disabling the guard", async () => { + vi.stubEnv("PUMPERLY_OCPDB_MIN_STATIONS", "not-a-number"); + const { queries, scraper } = await setup({ results: [[{ count: BigInt(42) }]] }); + + await scraper.run(); + + expect(queries).toHaveLength(1); + }); +}); diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts new file mode 100644 index 0000000..ab83f64 --- /dev/null +++ b/src/scrapers/ocpdb.ts @@ -0,0 +1,648 @@ +import { z } from "zod"; +import type { EvFees, SessionFeeRule } from "../types/station"; +import { + BaseScraper, + isPlausibleEvPrice, + maxSanePowerKw, + type RawFuelPrice, + type RawStation, + type ScraperResult, +} from "./base"; +import { otherGermanyEvPrefixes } from "./germany-ev-source"; + +// --------------------------------------------------------------------------- +// OCPDB (Open ChargePoint Database) via MobiData BW — German EV chargers + prices +// --------------------------------------------------------------------------- +// API: https://api.mobidata-bw.de/ocpdb/api/ocpi/3.0/{locations,tariffs,tariff-associations} +// (the central api.ocpdb.de only serves the map UI; its OCPI paths 404) +// Sources: the BNetzA register (`bnetza_api`) merged with the operators' own +// DATEX II feeds from Mobilithek (EnBW, EcoMovement, Chargecloud, +// Wirelane, e-clearing.net, Tesla, ...). Freshness per source: +// /ocpdb/api/public/v1/sources +// Licence: BNetzA and EnBW are CC BY 4.0, the other feeds CC0 (see +// components/nav/legal-modal.tsx). No API key, no signup. +// +// An alternative to the BNetzA scraper for Germany's EV chargers +// (PUMPERLY_DE_EV_SOURCE=ocpdb): same register, plus ad-hoc tariffs for ~97k +// EVSEs, which BNetzA does not publish. +// +// THREE THINGS SHAPE THIS FILE. +// +// 1. The feeds overlap. A charger the operator reports over DATEX II is often +// also in the register, under a different operator name (site owner vs +// CPO) and with coordinates a few metres off. Measured on 2026-10-22: +// ~4.7k EcoMovement and ~4.9k Chargecloud locations sit within 25 m of a +// register location, many at 0 m. Hence mergeLocations(): a 25 m greedy +// merge, register first so its coordinates anchor the pins. +// +// 2. VAT is inconsistent across feeds. EcoMovement and EnBW publish net +// prices with `taxes: [VAT 19%]` and `tax_included: false`, Wirelane +// publishes gross, Chargecloud publishes no tax data at all. Ad-hoc +// charging is a consumer price and PAngV requires it gross, so "no tax +// data" is read as gross. A price marked net with no rate gets German +// VAT. See grossPrice(). +// +// 3. TIME components read as per minute here, not OCPI's per hour. Every +// one seen is a blocking fee starting 30-240 min in (EnBW 0.2017 net +// after 30 min, Wirelane 0.20 after 45 min, Chargecloud 0.06 after +// 4 h). Those match the per-minute blocking fees operators advertise. +// Per hour they would be too small to block anyone. +// +// Like bnetza.ts, run() sweeps afterwards, because base.run() never +// garbage-collects EV rows. +// --------------------------------------------------------------------------- + +const BASE_URL = "https://api.mobidata-bw.de/ocpdb/api/ocpi/3.0"; + +// Hard upstream cap: a larger limit is a 400. +const PAGE_SIZE = 1000; +const CONCURRENCY = 4; +const PAGE_TIMEOUT_MS = 120_000; + +// Germany's bounding box, padded. Same as bnetza.ts. +const DE_BBOX = { latMin: 47, latMax: 56, lonMin: 5.5, lonMax: 15.5 }; + +/** Locations closer than this fold into one pin. */ +const MERGE_RADIUS_M = 25; +// Grid cell for the merge lookup, in degrees. ~111 m of latitude, so a 3×3 +// neighbourhood always covers the merge radius. +const GRID_DEG = 0.001; + +// Baked into every externalId. Changing it re-mints every row. +const COORD_PRECISION = 5; + +// Lower rank wins the anchor. The register first: its coordinates are the +// official ones and it covers every operator. Then the feeds that carry +// prices, so a priced pin keeps the operator's own name when the register has +// no entry nearby. +const SOURCE_RANK: Record = { + bnetza_api: 0, + datex2_enbw: 1, + datex2_ecomovement: 2, + datex2_chargecloud: 3, + datex2_wirelane: 4, +}; +const DEFAULT_RANK = 9; + +// EVSEs in these states are not a charger anyone can drive to. +const DEAD_EVSE_STATUSES = new Set(["REMOVED", "PLANNED"]); + +// German standard VAT, for prices marked net that don't say at what rate. +const DE_VAT_PCT = 19; + +// Session size used to fold a session fee into a per-kWh figure when ranking +// tariffs: a typical fast-charge top-up. +const REF_SESSION_KWH = 30; + +// A feed whose static data hasn't refreshed in this long, or that OCPDB marks +// as failed, has its tariffs ignored: its last prices would otherwise stay on +// the map indefinitely. Per-source, not per-tariff: a tariff's last_updated is +// when its price last changed, and most valid ones are months old. +const rawMaxFeedAge = Number(process.env.PUMPERLY_OCPDB_MAX_FEED_AGE_DAYS ?? "3"); +const MAX_FEED_AGE_DAYS = Number.isFinite(rawMaxFeedAge) && rawMaxFeedAge > 0 ? rawMaxFeedAge : 3; + +// Same floor as bnetza.ts, for the same reasons: a degraded fetch +// must not be able to bulk-delete yesterday's map or retire the other sources. +const rawMinStations = Number(process.env.PUMPERLY_OCPDB_MIN_STATIONS ?? "10000"); +const MIN_STATIONS = + Number.isFinite(rawMinStations) && rawMinStations >= 1 ? Math.floor(rawMinStations) : 10_000; + +// --------------------------------------------------------------------------- +// Upstream shapes. Only the fields used; everything else passes through. +// --------------------------------------------------------------------------- + +const TaxSchema = z.object({ percentage: z.coerce.number() }); + +const PriceComponentSchema = z.object({ + type: z.string(), + price: z.number(), + taxes: z.array(TaxSchema).optional(), + tax_included: z.boolean().optional(), +}); + +const TariffSchema = z.object({ + id: z.string(), + source: z.string(), + currency: z.string(), + tax_included: z.string().nullish(), + elements: z.array( + z.object({ + price_components: z.array(PriceComponentSchema), + restrictions: z + .object({ + min_duration: z.number().nullish(), + // Only shown as text, so an odd value drops the detail, not the tariff. + start_time: z.string().nullish().catch(null), + end_time: z.string().nullish().catch(null), + day_of_week: z.array(z.string()).nullish().catch(null), + }) + .passthrough() + .nullish(), + }), + ), +}); + +const AssociationSchema = z.object({ + tariff_id: z.string(), + audience: z.string().nullish(), + evses: z.array(z.object({ evse_uid: z.string() })).default([]), +}); + +const LocationSchema = z.object({ + source: z.string(), + country: z.string(), + name: z.string().nullish(), + address: z.string().nullish(), + postal_code: z.string().nullish(), + city: z.string().nullish(), + state: z.string().nullish(), + coordinates: z.object({ latitude: z.number(), longitude: z.number() }), + operator: z.object({ name: z.string().nullish() }).nullish(), + charging_pool: z + .array( + z.object({ + evses: z + .array( + z.object({ + uid: z.string(), + status: z.string().nullish(), + connectors: z + .array(z.object({ max_electric_power: z.number().nullish() })) + .default([]), + }), + ) + .default([]), + }), + ) + .default([]), +}); + +const SourceSchema = z.object({ + uid: z.string(), + static_status: z.string().nullish(), + static_data_updated_at: z.string().nullish(), +}); + +export type OcpdbSource = z.infer; +export type OcpdbTariff = z.infer; +export type OcpdbAssociation = z.infer; +export type OcpdbLocation = z.infer; +type PriceComponent = z.infer; + +// --------------------------------------------------------------------------- +// Tariffs +// --------------------------------------------------------------------------- + +export interface TariffSummary { + /** Gross price per kWh. */ + energy: number; + currency: string; + fees: EvFees; + /** When the tariff's feed last refreshed, if known. */ + reportedAt?: Date; +} + +/** A component's price with VAT, rounded to 4 dp. See header note 2. */ +export function grossPrice(component: PriceComponent, tariff: Pick): number { + const included = component.tax_included ?? (tariff.tax_included === "NO" ? false : true); + // Marked net but no rate given: the price is still net, so add German VAT. + const vat = included + ? 0 + : component.taxes?.length + ? component.taxes.reduce((sum, t) => sum + t.percentage, 0) + : DE_VAT_PCT; + return Math.round(component.price * (1 + vat / 100) * 10_000) / 10_000; +} + +/** + * What ranks tariffs against each other: €/kWh with the session fee spread + * over a typical session, so a low rate can't hide a large fee. + */ +export function effectivePerKwh(s: TariffSummary): number { + return s.energy + (s.fees.sessionFee ?? 0) / REF_SESSION_KWH; +} + +function cheaper(a: TariffSummary, b: TariffSummary | null | undefined): boolean { + return !b || effectivePerKwh(a) < effectivePerKwh(b); +} + +/** + * Reduce a tariff to what a pin can show: gross €/kWh plus the fees on top. + * Null when the tariff has no per-kWh price (a flat- or time-only tariff + * cannot be ranked against the rest). + */ +export function summarizeTariff(tariff: OcpdbTariff): TariffSummary | null { + let energy: number | null = null; + // FLAT fees in unrestricted elements always apply, so they add up. Restricted + // ones (by time of day, weekday...) are alternatives: each becomes a rule the + // popup lists, and the highest fee is the one cost estimates use. + let sessionFee = 0; + const rules: SessionFeeRule[] = []; + let perMinute: { price: number; afterSec: number } | null = null; + + for (const element of tariff.elements) { + const afterSec = element.restrictions?.min_duration ?? 0; + const restricted = Object.values(element.restrictions ?? {}).some((v) => v != null); + for (const c of element.price_components) { + const price = grossPrice(c, tariff); + if (c.type === "ENERGY") { + // Restricted energy prices (by duration, power...) are the exception; + // the first one listed is the headline rate. + if (energy == null) energy = price; + } else if (c.type === "FLAT") { + if (!restricted) sessionFee += price; + else rules.push(feeRule(price, element.restrictions ?? {})); + } else if (c.type === "TIME" && price > 0) { + // Several TIME steps: the first one that costs anything is the fee + // a driver will actually meet. + if (perMinute == null || afterSec < perMinute.afterSec) perMinute = { price, afterSec }; + } + } + } + + if (energy == null) return null; + const fees: EvFees = {}; + const highest = Math.max(sessionFee, ...rules.map((r) => r.fee)); + if (highest > 0) { + fees.sessionFee = round4(highest); + // A restricted fee equal to the always-on one changes nothing (some feeds + // repeat their elements under a min_duration), so it isn't listed. + const always = round4(sessionFee); + const varying = rules.filter((r) => r.fee !== always); + if (varying.length > 0) fees.sessionFeeRules = always > 0 ? [{ fee: always }, ...varying] : varying; + } + if (perMinute) { + fees.perMinute = perMinute.price; + fees.afterMin = Math.round(perMinute.afterSec / 60); + } + return { energy, currency: tariff.currency, fees }; +} + +const round4 = (n: number) => Math.round(n * 10_000) / 10_000; + +const ISO_DAYS = ["MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY", "SATURDAY", "SUNDAY"]; +const SHOWN_RESTRICTIONS = new Set(["start_time", "end_time", "day_of_week"]); + +/** When a restricted session fee applies, in the shape the popup prints. */ +function feeRule(price: number, r: NonNullable): SessionFeeRule { + const rule: SessionFeeRule = { fee: round4(price) }; + if (r.start_time) rule.from = r.start_time; + if (r.end_time) rule.to = r.end_time; + const days = (r.day_of_week ?? []).map((d) => ISO_DAYS.indexOf(d) + 1).filter((d) => d > 0); + if (days.length > 0) rule.days = days; + if (Object.entries(r).some(([k, v]) => v != null && !SHOWN_RESTRICTIONS.has(k))) rule.other = true; + return rule; +} + +/** + * The feeds whose tariffs can be trusted, each with when it last refreshed: + * OCPDB reports them ACTIVE and their static data (which carries the tariffs) + * refreshed within `maxAgeDays`. + */ +export function liveSources(sources: OcpdbSource[], now: Date, maxAgeDays: number): Map { + const cutoff = now.getTime() - maxAgeDays * 86_400_000; + const live = new Map(); + for (const s of sources) { + const updated = Date.parse(s.static_data_updated_at ?? ""); + if (s.static_status === "ACTIVE" && updated >= cutoff) live.set(s.uid, new Date(updated)); + } + return live; +} + +/** + * EVSE uid → cheapest ad-hoc tariff on it. Only AD_HOC_PAYMENT counts (an + * association with no audience doesn't): that is the price a driver without a + * contract pays, the only one comparable across operators. Tariffs the map + * would reject (see isPlausibleEvPrice) are dropped before ranking, so a bogus + * 0.00 can't hide a valid price. With `live`, tariffs from any other feed are + * dropped too, and the rest carry their feed's refresh time. + */ +export function evseTariffs( + tariffs: OcpdbTariff[], + associations: OcpdbAssociation[], + live?: Map, +): Map { + const byId = new Map(); + for (const t of tariffs) { + if (live && !live.has(t.source)) continue; + const s = summarizeTariff(t); + if (!s || !isPlausibleEvPrice(s.energy, s.currency)) continue; + const reportedAt = live?.get(t.source); + byId.set(t.id, reportedAt ? { ...s, reportedAt } : s); + } + const byEvse = new Map(); + for (const a of associations) { + if (a.audience !== "AD_HOC_PAYMENT") continue; + const s = byId.get(a.tariff_id); + if (!s) continue; + for (const { evse_uid } of a.evses) { + if (cheaper(s, byEvse.get(evse_uid))) byEvse.set(evse_uid, s); + } + } + return byEvse; +} + +// --------------------------------------------------------------------------- +// Locations +// --------------------------------------------------------------------------- + +/** The merge anchor's key, which is also the externalId. */ +export function stationKey(latitude: number, longitude: number): string { + return `ocpdb-${latitude.toFixed(COORD_PRECISION)}_${longitude.toFixed(COORD_PRECISION)}`; +} + +function distanceM(lat1: number, lon1: number, lat2: number, lon2: number): number { + const dLat = (lat2 - lat1) * 111_320; + const dLon = (lon2 - lon1) * 111_320 * Math.cos((lat1 * Math.PI) / 180); + return Math.hypot(dLat, dLon); +} + +export interface OcpdbParseStats { + locations: number; + notGermany: number; + noLiveEvse: number; + merged: number; + priced: number; +} + +interface Pin { + station: RawStation; + tariff: TariffSummary | null; + /** The location's own name or street, which the pin's name is built from. */ + site: string; +} + +/** "Operator — site", falling back to whichever part exists, then the key. */ +function pinName(operator: string | null, site: string, key: string): string { + return [operator, site].filter(Boolean).join(" — ") || key; +} + +/** + * Turn OCPDB locations into deduplicated pins, each with its cheapest ad-hoc + * tariff. A pin is labelled with the operator whose tariff it shows: when a + * merged location supplies the price, its operator takes over the brand, so a + * driver never sees one operator's name next to another's price. Pure: the merge, the filters and the derived externalId all live + * here, so this is what the tests cover. + */ +export function mergeLocations( + locations: OcpdbLocation[], + tariffsByEvse: Map, +): { stations: RawStation[]; prices: RawFuelPrice[]; stats: OcpdbParseStats } { + const stats: OcpdbParseStats = { locations: locations.length, notGermany: 0, noLiveEvse: 0, merged: 0, priced: 0 }; + + // Stable sort: equal ranks keep upstream order, so the anchors (and the + // externalIds) don't shuffle between runs. + const ordered = locations + .map((loc, i) => ({ loc, i, rank: SOURCE_RANK[loc.source] ?? DEFAULT_RANK })) + .sort((a, b) => a.rank - b.rank || a.i - b.i) + .map((x) => x.loc); + + const grid = new Map(); + const pins: Pin[] = []; + const cell = (lat: number, lon: number) => `${Math.floor(lat / GRID_DEG)}:${Math.floor(lon / GRID_DEG)}`; + + for (const loc of ordered) { + const { latitude, longitude } = loc.coordinates; + if ( + loc.country !== "DEU" || + latitude < DE_BBOX.latMin || + latitude > DE_BBOX.latMax || + longitude < DE_BBOX.lonMin || + longitude > DE_BBOX.lonMax + ) { + stats.notGermany++; + continue; + } + + const evses = loc.charging_pool + .flatMap((p) => p.evses) + .filter((e) => !DEAD_EVSE_STATUSES.has(e.status ?? "")); + if (evses.length === 0) { + stats.noLiveEvse++; + continue; + } + + // max_electric_power is in W. + const powerKw = maxSanePowerKw( + evses.flatMap((e) => e.connectors.map((c) => (c.max_electric_power ?? 0) / 1000)), + ); + let tariff: TariffSummary | null = null; + for (const e of evses) { + const t = tariffsByEvse.get(e.uid); + if (t && cheaper(t, tariff)) tariff = t; + } + const operator = loc.operator?.name?.trim() || null; + + // Fold into an existing pin within the merge radius, if any. + const gy = Math.floor(latitude / GRID_DEG); + const gx = Math.floor(longitude / GRID_DEG); + let target: Pin | null = null; + let best = MERGE_RADIUS_M; + for (let dy = -1; dy <= 1; dy++) { + for (let dx = -1; dx <= 1; dx++) { + for (const pin of grid.get(`${gy + dy}:${gx + dx}`) ?? []) { + const d = distanceM(pin.station.latitude, pin.station.longitude, latitude, longitude); + if (d <= best) { + best = d; + target = pin; + } + } + } + } + + if (target) { + stats.merged++; + if (powerKw != null && powerKw > (target.station.maxPowerKw ?? 0)) target.station.maxPowerKw = powerKw; + if (tariff && cheaper(tariff, target.tariff)) { + target.tariff = tariff; + if (operator) { + target.station.brand = operator; + target.station.name = pinName(operator, target.site, target.station.externalId); + } + } + continue; + } + + const key = stationKey(latitude, longitude); + const street = loc.address?.trim() ?? ""; + const postcode = loc.postal_code?.trim() ?? ""; + const site = loc.name?.trim() || street; + const name = pinName(operator, site, key); + const pin: Pin = { + station: { + externalId: key, + name, + brand: operator, + address: [street, postcode].filter(Boolean).join(", ") || name, + city: loc.city?.trim() ?? "", + province: loc.state?.trim() || null, + latitude, + longitude, + stationType: "ev_charger", + maxPowerKw: powerKw, + }, + tariff, + site, + }; + pins.push(pin); + const k = cell(latitude, longitude); + const bucket = grid.get(k); + if (bucket) bucket.push(pin); + else grid.set(k, [pin]); + } + + const prices: RawFuelPrice[] = []; + for (const { station, tariff } of pins) { + if (!tariff) continue; + stats.priced++; + prices.push({ + stationExternalId: station.externalId, + fuelType: "EV", + price: tariff.energy, + currency: tariff.currency, + ...(Object.keys(tariff.fees).length > 0 ? { evFees: tariff.fees } : {}), + ...(tariff.reportedAt ? { reportedAt: tariff.reportedAt } : {}), + }); + } + + return { stations: pins.map((p) => p.station), prices, stats }; +} + +// --------------------------------------------------------------------------- +// Fetch +// --------------------------------------------------------------------------- + +const PageSchema = z.object({ + items: z.array(z.unknown()), + total_count: z.number(), +}); + +async function fetchPage(path: string, offset: number): Promise> { + const sep = path.includes("?") ? "&" : "?"; + const url = `${BASE_URL}/${path}${sep}limit=${PAGE_SIZE}&offset=${offset}`; + let lastErr: unknown; + // One retry: a single slow page should not cost the whole run. + for (let attempt = 0; attempt < 2; attempt++) { + try { + const res = await fetch(url, { + headers: { Accept: "application/json", "User-Agent": "Pumperly/1.0 (+https://pumperly.com)" }, + signal: AbortSignal.timeout(PAGE_TIMEOUT_MS), + }); + if (!res.ok) throw new Error(`OCPDB HTTP ${res.status} for ${path} @${offset}`); + return PageSchema.parse(await res.json()); + } catch (err) { + lastErr = err; + } + } + throw lastErr; +} + +/** + * Every item of a paginated collection, each parsed with `schema`. Items that + * fail to parse are counted and skipped, not fatal: one odd location must not + * cost the country. + */ +async function fetchAll(path: string, schema: z.ZodType): Promise<{ items: T[]; invalid: number }> { + const first = await fetchPage(path, 0); + const offsets: number[] = []; + for (let o = PAGE_SIZE; o < first.total_count; o += PAGE_SIZE) offsets.push(o); + + const pages: unknown[][] = [first.items]; + for (let i = 0; i < offsets.length; i += CONCURRENCY) { + const batch = await Promise.all(offsets.slice(i, i + CONCURRENCY).map((o) => fetchPage(path, o))); + for (const p of batch) pages.push(p.items); + } + + const items: T[] = []; + let invalid = 0; + for (const page of pages) { + for (const raw of page) { + const r = schema.safeParse(raw); + if (r.success) items.push(r.data); + else invalid++; + } + } + return { items, invalid }; +} + +/** + * Feed health from OCPDB's own status endpoint (not paginated, ~15 rows). + * Retried once like a page: a failure here costs the whole run. + */ +async function fetchSources(): Promise { + let lastErr: unknown; + for (let attempt = 0; attempt < 2; attempt++) { + try { + const res = await fetch("https://api.mobidata-bw.de/ocpdb/api/public/v1/sources", { + headers: { Accept: "application/json", "User-Agent": "Pumperly/1.0 (+https://pumperly.com)" }, + signal: AbortSignal.timeout(PAGE_TIMEOUT_MS), + }); + if (!res.ok) throw new Error(`OCPDB HTTP ${res.status} for sources`); + return z.object({ items: z.array(SourceSchema) }).parse(await res.json()).items; + } catch (err) { + lastErr = err; + } + } + throw lastErr; +} + +export class OcpdbScraper extends BaseScraper { + readonly country = "DE"; + readonly source = "ocpdb"; + + async fetch(): Promise<{ stations: RawStation[]; prices: RawFuelPrice[] }> { + const [sources, tariffs, associations, locations] = await Promise.all([ + // Without feed health no tariff can be vouched for, and a run without + // prices would wipe the current ones: failing here keeps them. + fetchSources(), + fetchAll("tariffs", TariffSchema), + fetchAll("tariff-associations", AssociationSchema), + fetchAll("locations?country=DEU", LocationSchema), + ]); + const live = liveSources(sources, new Date(), MAX_FEED_AGE_DAYS); + const stale = [...new Set(tariffs.items.map((t) => t.source))].filter((s) => !live.has(s)); + if (stale.length > 0) { + console.warn( + `[${this.source}] DE: ignoring tariffs from ${stale.join(", ")} ` + + `(failed, or no refresh in ${MAX_FEED_AGE_DAYS} days)`, + ); + } + const tariffsByEvse = evseTariffs(tariffs.items, associations.items, live); + const { stations, prices, stats } = mergeLocations(locations.items, tariffsByEvse); + + console.log( + `[${this.source}] DE: ${tariffs.items.length} tariffs → ${tariffsByEvse.size} priced EVSEs; ` + + `${stats.locations} locations → ${stations.length} stations, ${stats.priced} priced ` + + `(merged ${stats.merged}, skipped ${stats.noLiveEvse} without live EVSE, ${stats.notGermany} outside DE, ` + + `${tariffs.invalid + associations.invalid + locations.invalid} unparseable)`, + ); + + return { stations, prices }; + } + + /** + * Run the normal pipeline, then sweep (see BaseScraper.sweepEvRows): stale + * `ocpdb-` rows, and the other German EV sources' rows this one replaces. + * Germany has exactly one EV source at a time (germany-ev-source.ts), so + * once OCPDB has landed those are pure duplicates. + */ + async run(): Promise { + const result = await super.run(); + if (result.errors.length > 0) return result; + + try { + await this.sweepEvRows({ + ownPrefix: "ocpdb-", + retirePrefixes: otherGermanyEvPrefixes("ocpdb-"), + minStations: MIN_STATIONS, + runDurationMs: result.durationMs, + label: "OCPDB", + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.error(`[${this.source}] Cleanup failed: ${msg}`); + result.errors.push(`Cleanup: ${msg}`); + } + return result; + } +} From 0f7011e9a72819583be56e8387c59e4ad123e75b Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:52:25 +0200 Subject: [PATCH 06/25] feat(scrapers): make OCPDB the default German EV source It contains the BNetzA register and is the only German source with prices. PUMPERLY_DE_EV_SOURCE=bnetza or =ocm switch back. On its first healthy run it retires the bnetza- and ocm- rows. --- .env.example | 17 ++++++++--------- src/instrumentation-node.ts | 4 ++-- src/scrapers/cli.ts | 4 ++-- src/scrapers/germany-ev-source.test.ts | 14 +++++++------- src/scrapers/germany-ev-source.ts | 9 ++++----- src/scrapers/ocpdb.ts | 5 ++--- 6 files changed, 25 insertions(+), 28 deletions(-) diff --git a/.env.example b/.env.example index 89b7251..d3da950 100644 --- a/.env.example +++ b/.env.example @@ -90,15 +90,14 @@ PUMPERLY_DEFAULT_COUNTRY=ES # Set to 0 to disable EV charger scraping (default: enabled when OCM key is set) # PUMPERLY_EV_ENABLED=1 -# Germany EV chargers: which source supplies them. Defaults to "bnetza", the -# BNetzA Ladesäulenregister — the official register every operator of a public -# charge point must file into, ~74,000 locations, no API key and no signup. -# "ocpdb" uses the Open ChargePoint Database via MobiData BW instead: that -# register merged with the operators' own feeds, plus ad-hoc €/kWh prices, also -# keyless. "ocm" keeps OpenChargeMap. Only one ever runs. ocpdb and bnetza -# retire the other sources' German rows on their first healthy run; ocm does -# not, so switching to it leaves the `ocpdb-`/`bnetza-` rows to delete by hand. -# PUMPERLY_DE_EV_SOURCE=bnetza +# Germany EV chargers: which source supplies them. Defaults to "ocpdb", the Open +# ChargePoint Database via MobiData BW: the BNetzA register merged with the +# operators' own feeds, plus ad-hoc €/kWh prices. No API key, no signup. +# "bnetza" uses the BNetzA Ladesäulenregister alone (no prices), "ocm" keeps +# OpenChargeMap. Only one ever runs. ocpdb and bnetza retire the other +# sources' German rows on their first healthy run; ocm does not, so switching +# to it leaves the `ocpdb-`/`bnetza-` rows behind to delete by hand. +# PUMPERLY_DE_EV_SOURCE=ocpdb # Stations a run must refresh before the OCPDB scraper is allowed to prune # stale rows or retire the BNetzA/OpenChargeMap German ones. diff --git a/src/instrumentation-node.ts b/src/instrumentation-node.ts index 995f6d7..1d4eeaa 100644 --- a/src/instrumentation-node.ts +++ b/src/instrumentation-node.ts @@ -207,9 +207,9 @@ export async function registerNode() { EV_US: () => new OCMScraper("US"), // Spain's official EV registry — supersedes EV_ES when a key is set (#121) EV_ES_REVE: () => new REVEScraper(), - // Germany's official EV registry — supersedes EV_DE by default, keyless + // Germany's official EV registry — keyless, PUMPERLY_DE_EV_SOURCE=bnetza EV_DE_BNETZA: () => new BNetzAScraper(), - // Germany's register + operator feeds with ad-hoc prices — opt-in, PUMPERLY_DE_EV_SOURCE=ocpdb + // Germany's register + operator feeds with ad-hoc prices — the default DE EV source EV_DE_OCPDB: () => new OcpdbScraper(), }; diff --git a/src/scrapers/cli.ts b/src/scrapers/cli.ts index 9582829..526c144 100644 --- a/src/scrapers/cli.ts +++ b/src/scrapers/cli.ts @@ -144,9 +144,9 @@ const SCRAPERS: Record BaseScraper>> = { EV_US: [() => new OCMScraper("US")], // Spain's official EV registry — supersedes EV_ES when a key is set (#121) EV_ES_REVE: [() => new REVEScraper()], - // Germany's official EV registry — supersedes EV_DE by default, keyless + // Germany's official EV registry — keyless, PUMPERLY_DE_EV_SOURCE=bnetza EV_DE_BNETZA: [() => new BNetzAScraper()], - // Germany's register + operator feeds with ad-hoc prices — opt-in, PUMPERLY_DE_EV_SOURCE=ocpdb + // Germany's register + operator feeds with ad-hoc prices — the default DE EV source EV_DE_OCPDB: [() => new OcpdbScraper()], }; diff --git a/src/scrapers/germany-ev-source.test.ts b/src/scrapers/germany-ev-source.test.ts index d2cfe28..4b36b87 100644 --- a/src/scrapers/germany-ev-source.test.ts +++ b/src/scrapers/germany-ev-source.test.ts @@ -17,13 +17,13 @@ describe("resolveGermanyEvSource", () => { const GERMAN = ["EV_DE", "EV_DE_BNETZA", "EV_DE_OCPDB"]; const german = (codes: string[]) => codes.filter((c) => GERMAN.includes(c)); - it("uses BNetzA for Germany by default", () => { - expect(german(resolveGermanyEvSource(SCHEDULED))).toEqual(["EV_DE_BNETZA"]); + it("uses OCPDB for Germany by default", () => { + expect(german(resolveGermanyEvSource(SCHEDULED))).toEqual(["EV_DE_OCPDB"]); }); - it("uses OCPDB when PUMPERLY_DE_EV_SOURCE=ocpdb", () => { - vi.stubEnv("PUMPERLY_DE_EV_SOURCE", "ocpdb"); - expect(german(resolveGermanyEvSource(SCHEDULED))).toEqual(["EV_DE_OCPDB"]); + it("uses BNetzA when PUMPERLY_DE_EV_SOURCE=bnetza", () => { + vi.stubEnv("PUMPERLY_DE_EV_SOURCE", "bnetza"); + expect(german(resolveGermanyEvSource(SCHEDULED))).toEqual(["EV_DE_BNETZA"]); }); it("keeps OpenChargeMap when PUMPERLY_DE_EV_SOURCE=ocm", () => { @@ -36,10 +36,10 @@ describe("resolveGermanyEvSource", () => { expect(resolveGermanyEvSource(SCHEDULED)).toContain("EV_DE"); }); - it("falls back to BNetzA when the override is not a source it knows", () => { + it("falls back to OCPDB when the override is not a source it knows", () => { for (const source of ["nonsense", "constructor", "toString"]) { vi.stubEnv("PUMPERLY_DE_EV_SOURCE", source); - expect(german(resolveGermanyEvSource(SCHEDULED))).toEqual(["EV_DE_BNETZA"]); + expect(german(resolveGermanyEvSource(SCHEDULED))).toEqual(["EV_DE_OCPDB"]); } }); diff --git a/src/scrapers/germany-ev-source.ts b/src/scrapers/germany-ev-source.ts index 6cf879f..c589a8a 100644 --- a/src/scrapers/germany-ev-source.ts +++ b/src/scrapers/germany-ev-source.ts @@ -13,9 +13,8 @@ // `ocm-` rows (see scrapers/bnetza.ts), a single stray OpenChargeMap run puts // every one of them straight back, and nothing garbage-collects EV rows. // -// BNetzA wins by default: it is official, it needs no key, and it is the more -// complete of the register-based sources. `PUMPERLY_DE_EV_SOURCE=ocpdb` adds -// prices; `=ocm` keeps OpenChargeMap. +// OCPDB wins by default: it is the only one with prices, and it contains the +// register. `PUMPERLY_DE_EV_SOURCE=bnetza` or `=ocm` are the escape hatches. // The OCPDB and BNetzA sweeps retire the other sources' rows (see // scrapers/ocpdb.ts and scrapers/bnetza.ts); OpenChargeMap has no sweep, so // switching to it leaves the `ocpdb-`/`bnetza-` rows to delete by hand. @@ -48,7 +47,7 @@ export function otherGermanyEvPrefixes(own: GermanyEvPrefix): string[] { * * Returns `codes` unchanged when Germany has no EV scraper enabled at all (e.g. * PUMPERLY_EV_ENABLED=0). Otherwise exactly one survives: the one named by - * PUMPERLY_DE_EV_SOURCE (ocm | bnetza | ocpdb), BNetzA when unset or unknown. + * PUMPERLY_DE_EV_SOURCE (ocm | bnetza | ocpdb), OCPDB when unset or unknown. */ export function resolveGermanyEvSource(codes: string[]): string[] { const all = Object.values(GERMANY_EV_CODES); @@ -56,6 +55,6 @@ export function resolveGermanyEvSource(codes: string[]): string[] { const rest = codes.filter((c) => !all.includes(c)); if (!hasGermanyEv) return rest; const chosen = process.env.PUMPERLY_DE_EV_SOURCE?.trim().toLowerCase() ?? ""; - rest.push(Object.hasOwn(GERMANY_EV_CODES, chosen) ? GERMANY_EV_CODES[chosen] : GERMANY_EV_CODES.bnetza); + rest.push(Object.hasOwn(GERMANY_EV_CODES, chosen) ? GERMANY_EV_CODES[chosen] : GERMANY_EV_CODES.ocpdb); return rest; } diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts index ab83f64..b778787 100644 --- a/src/scrapers/ocpdb.ts +++ b/src/scrapers/ocpdb.ts @@ -22,9 +22,8 @@ import { otherGermanyEvPrefixes } from "./germany-ev-source"; // Licence: BNetzA and EnBW are CC BY 4.0, the other feeds CC0 (see // components/nav/legal-modal.tsx). No API key, no signup. // -// An alternative to the BNetzA scraper for Germany's EV chargers -// (PUMPERLY_DE_EV_SOURCE=ocpdb): same register, plus ad-hoc tariffs for ~97k -// EVSEs, which BNetzA does not publish. +// Replaces the BNetzA scraper as Germany's default EV source: same register, +// plus ad-hoc tariffs for ~97k EVSEs, which BNetzA does not publish. // // THREE THINGS SHAPE THIS FILE. // From 573cc05ff4b69b7a99415f30d0770aa98d065cd7 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:53:55 +0200 Subject: [PATCH 07/25] feat(planner): price EV plans with kWh and session fee MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When chargers on the route have prices, the EV planner only considers priced chargers. Each stop costs kWh × price + the session fee, using the highest fee when it varies. The plan shows a cost per stop and in total, and the value-of-time input appears. Routes with no priced chargers keep the time-only plan. --- src/components/search/refuel-planner.test.tsx | 28 ++++++++++++ src/components/search/refuel-planner.tsx | 43 +++++++++++-------- src/lib/refuel-planner.test.ts | 15 +++++++ src/lib/refuel-planner.ts | 15 ++++--- 4 files changed, 77 insertions(+), 24 deletions(-) diff --git a/src/components/search/refuel-planner.test.tsx b/src/components/search/refuel-planner.test.tsx index ade6547..c2047de 100644 --- a/src/components/search/refuel-planner.test.tsx +++ b/src/components/search/refuel-planner.test.tsx @@ -415,6 +415,34 @@ describe("RefuelPlanner", () => { expect(screen.getAllByText(/^\d+ (h \d+ )?min · /).length).toBeGreaterThan(0); }); + it("plans by price when chargers have one, with the session fee per stop", async () => { + const chargers = [ + makeStation("x", { brand: "Ionity", price: 0.69, fuelType: "EV", routeFraction: 0.3, detourMin: 0, evFees: { sessionFee: 1.5 } }), + makeStation("y", { brand: "Tesla", price: 0.49, fuelType: "EV", routeFraction: 0.6, detourMin: 0 }), + makeStation("z", { brand: "Unpriced", price: null, currency: undefined, fuelType: "EV", routeFraction: 0.45, detourMin: 0 }), + ]; + renderPlanner({ mode: "ev", stations: chargers, routeKm: 300 }); + await userEvent.click(screen.getByText("planner.titleEv")); + const input = planSpy.mock.lastCall![0]; + // Priced chargers only, at their price, with the real value of time. + expect(input.stations.map((s: { id: string }) => s.id)).toEqual(["x", "y"]); + expect(input.stations[0]).toMatchObject({ price: 0.69, stopFee: 1.5 }); + expect(input.stations[1].stopFee).toBeUndefined(); + expect(input.timeValuePerHour).toBe(15); + expect(screen.getByText(/planner.timeValue/)).toBeInTheDocument(); + expect(screen.getAllByText(/^\d+\.\d\d € · \+\d+\.\d kWh$/)).toHaveLength(2); + }); + + it("converts the session fee from the tariff's currency", async () => { + Object.assign(currencyState, { currency: "HUF", symbol: "Ft", decimals: 0, rate: 400, rates: { base: "EUR", rates: { HUF: 400 }, date: "2026-10-22" } }); + const chargers = [ + makeStation("x", { price: 276, currency: "HUF", originalPrice: 0.69, originalCurrency: "EUR", fuelType: "EV", routeFraction: 0.3, detourMin: 0, evFees: { sessionFee: 1.5 } }), + ]; + renderPlanner({ mode: "ev", stations: chargers, routeKm: 300 }); + await userEvent.click(screen.getByText("planner.titleEv")); + expect(planSpy.mock.lastCall![0].stations[0]).toMatchObject({ price: 276, stopFee: 600 }); + }); + it("drops slower and unknown-power chargers under a minimum power", async () => { const chargers = [ makeStation("fast", { price: null, fuelType: "EV", routeFraction: 0.3, powerKw: 150 }), diff --git a/src/components/search/refuel-planner.tsx b/src/components/search/refuel-planner.tsx index 7b9dc5b..b1362d3 100644 --- a/src/components/search/refuel-planner.tsx +++ b/src/components/search/refuel-planner.tsx @@ -19,7 +19,7 @@ const EV_MAX_CHARGE_PCT = 80; const MIN_CHARGER_KW_OPTIONS = [0, 50, 150] as const; interface RefuelPlannerProps { - /** `ev` plans by energy only (kWh): chargers have no prices yet. */ + /** `ev` plans in kWh, by price when chargers on the route have one, else by time only. */ mode?: "fuel" | "ev"; /** Corridor stations (with routeFraction and, once streamed, detourMin). */ stations: StationGeoJSON[]; @@ -99,10 +99,16 @@ export function RefuelPlanner({ // Same for the value of time; null shows the saved (and planned) value. const [timeDraft, setTimeDraft] = useState(null); + // EV plans use money only when some charger on the route has a price in the + // display currency; then only priced chargers compete. Otherwise (no feed + // prices the chargers there) they compete on detour and charging time only. + const evPriced = ev && stations.some((s) => s.properties.price != null && s.properties.price > 0 && s.properties.currency === currency); + const usesMoney = !ev || evPriced; + // The time value is stored in EUR: without a rate for the display currency it // can't be converted, and `convert` would silently treat it as 1:1. - // EV plans ignore money, so they don't need rates. - const ratesMissing = !ev && currency !== "EUR" && !rates?.rates[currency]; + // Plans that ignore money don't need rates. + const ratesMissing = usesMoney && currency !== "EUR" && !rates?.rates[currency]; const eurRate = convert(1, "EUR"); const timeValue = Number((timeValueEur * eurRate).toFixed(Math.min(decimals, 2))); @@ -114,26 +120,24 @@ export function RefuelPlanner({ // Usable stations exist but every detour request failed: filters are not the cause. const detoursFailed = useMemo(() => { - const usable = ev ? stations : stations.filter((s) => s.properties.price != null); + const usable = ev && !evPriced ? stations : stations.filter((s) => s.properties.price != null); return usable.length > 0 && usable.every((s) => s.properties.detourMin != null && s.properties.detourMin < 0); - }, [ev, stations]); + }, [ev, evPriced, stations]); const plan: PlanResult | null = useMemo(() => { if (!open || ratesMissing || detoursPending || routeKm <= 0) return null; const candidates = stations.flatMap((s) => { const p = s.properties; if (ev) { - // No prices yet: chargers compete on detour and charging time only. if (p.detourMin == null || p.detourMin < 0 || (maxDetour != null && p.detourMin > maxDetour)) return []; // A power filter can't vouch for a charger whose power is unknown. if (minChargerKw > 0 && !((p.powerKw ?? 0) >= minChargerKw)) return []; - return [{ - id: p.id, - km: (p.routeFraction ?? 0) * routeKm, - price: 0, - detourMin: p.detourMin, - minutesPerPct: minutesPerPct(evProfile, p.powerKw), - }]; + const base = { id: p.id, km: (p.routeFraction ?? 0) * routeKm, detourMin: p.detourMin, minutesPerPct: minutesPerPct(evProfile, p.powerKw) }; + if (!evPriced) return [{ ...base, price: 0 }]; + if (p.price == null || p.price <= 0 || p.currency !== currency || (maxPrice != null && p.price > maxPrice)) return []; + // The session fee is in the tariff's own currency; the price is already converted. + const fee = p.evFees?.sessionFee; + return [{ ...base, price: p.price, ...(fee != null ? { stopFee: convert(fee, p.originalCurrency ?? p.currency) } : {}) }]; } // Mixed currencies can't be compared; only use prices in the display currency. if (p.price == null || p.price <= 0 || p.detourMin == null || p.detourMin < 0 || p.currency !== currency) return []; @@ -149,11 +153,11 @@ export function RefuelPlanner({ arrivalPct: arrival, reservePct, // With no price the cost is all time, so any positive value gives the same plan. - timeValuePerHour: ev ? 1 : timeValue, + timeValuePerHour: usesMoney ? timeValue : 1, maxStops: maxStopsFor(routeKm, capacity, consumption, reservePct, maxChargePct), maxChargePct, }); - }, [open, ratesMissing, detoursPending, routeKm, stations, ev, evProfile, minChargerKw, maxPrice, maxDetour, currency, capacity, consumption, maxChargePct, startPct, arrival, reservePct, timeValue]); + }, [open, ratesMissing, detoursPending, routeKm, stations, ev, evPriced, usesMoney, evProfile, minChargerKw, maxPrice, maxDetour, currency, convert, capacity, consumption, maxChargePct, startPct, arrival, reservePct, timeValue]); useEffect(() => { const stops = plan?.status === "ok" @@ -292,7 +296,7 @@ export function RefuelPlanner({ update({ startPct: v })} /> update({ arrivalPct: v })} /> update(ev ? { evReservePct: v } : { reservePct: v })} /> - {!ev &&
    diff --git a/src/lib/refuel-planner.test.ts b/src/lib/refuel-planner.test.ts index 7278887..4f2f472 100644 --- a/src/lib/refuel-planner.test.ts +++ b/src/lib/refuel-planner.test.ts @@ -431,6 +431,21 @@ describe("planRefuel with charge time (EV)", () => { expect(r.stops.map((s) => s.id)).toEqual(["ok"]); }); + it("adds a stop fee to the stop's cost and weighs it when choosing", () => { + const priced = (id: string, price: number, stopFee?: number): PlannerStation => ({ id, km: 100, price, detourMin: 0, minutesPerPct: 0.5, ...(stopFee != null ? { stopFee } : {}) }); + // 45 % of 60 kWh = 27 kWh: 0.39 × 27 + 10 = 20.53 loses to 0.45 × 27 = 12.15. + const r = planRefuel({ ...ev, routeKm: 250, stations: [priced("fee", 0.39, 10), priced("plain", 0.45)] }); + expect(r.stops.map((s) => s.id)).toEqual(["plain"]); + const only = planRefuel({ ...ev, routeKm: 250, stations: [priced("fee", 0.39, 10)] }); + expect(only.stops[0].cost).toBeCloseTo(0.39 * 27 + 10); + expect(only.totalFuelCost).toBeCloseTo(0.39 * 27 + 10); + }); + + it("ignores a station with an invalid stop fee", () => { + const r = planRefuel({ ...ev, routeKm: 250, stations: [{ ...at("bad", 100, 0, 1), stopFee: -1 }, at("ok", 100, 0, 1)] }); + expect(r.stops.map((s) => s.id)).toEqual(["ok"]); + }); + it("keeps the fastest charger in each bucket when pruning ties on price", () => { const stations = Array.from({ length: 400 }, (_, i) => at(`S${i}`, i * 5, 5, i === 201 ? 0.1 : 3)); expect(pruneCandidates(stations, 2000).map((s) => s.id)).toContain("S201"); diff --git a/src/lib/refuel-planner.ts b/src/lib/refuel-planner.ts index e8175ba..f09d44a 100644 --- a/src/lib/refuel-planner.ts +++ b/src/lib/refuel-planner.ts @@ -1,8 +1,8 @@ /** - * Refuel stop planner (liquid fuels). + * Refuel stop planner (liquid fuels and EV charging). * * Picks 0..maxStops corridor stations that minimise - * fuel bought · price + (detour + fixed stop time + charge time) · value of time + * fuel bought · price + per-stop fees + (detour + fixed stop time + charge time) · value of time * while the tank never drops below `reservePct` and arrives at the destination * with at least `arrivalPct`. The one exception is the first leg: when the start * level can't reach any station with the reserve intact, it may use up to half of @@ -19,8 +19,10 @@ export interface PlannerStation { id: string; /** Distance along the route from the origin, km. */ km: number; - /** Price per litre in the display currency. */ + /** Price per litre (per kWh for EV) in the display currency. */ price: number; + /** Fixed charge per stop in the display currency, e.g. an EV session fee. Unset = none. */ + stopFee?: number; /** Extra driving time to visit the station, minutes (≥ 0). */ detourMin: number; /** Time to add 1 % of capacity here, minutes (EV charging). Unset = instant (fuel). */ @@ -118,7 +120,7 @@ export function pruneCandidates(stations: PlannerStation[], routeKm: number, max // Where a full tank bought here effectively starts from, on the route. const onwardKm = (s: PlannerStation) => s.km - halfDetourKm(s); for (const list of byBucket.values()) { - // Ties go to the faster charger: EV prices are all 0, so there "cheapest" means "fastest". + // Ties go to the faster charger: unpriced EV plans set every price to 0, so there "cheapest" means "fastest". const cheapest = list.reduce((a, b) => b.price < a.price || (b.price === a.price && (b.minutesPerPct ?? 0) < (a.minutesPerPct ?? 0)) ? b : a); const closest = list.reduce((a, b) => (b.detourMin < a.detourMin ? b : a)); @@ -181,13 +183,14 @@ export function planRefuel(input: PlannerInput): PlanResult { .filter((s) => !seen.has(s.id) && seen.add(s.id)) .filter((s) => Number.isFinite(s.km) && Number.isFinite(s.price) && Number.isFinite(s.detourMin)) .filter((s) => s.minutesPerPct == null || (Number.isFinite(s.minutesPerPct) && s.minutesPerPct >= 0)) + .filter((s) => s.stopFee == null || (Number.isFinite(s.stopFee) && s.stopFee >= 0)) .filter((s) => s.km >= 0 && s.km <= routeKm && s.price >= 0 && s.detourMin >= 0) .sort((a, b) => a.km - b.km), routeKm, ); const n = stations.length; const detourKm = stations.map((s) => (s.detourMin / 60) * DETOUR_KMH); - const stopCost = stations.map((s) => ((s.detourMin + STOP_OVERHEAD_MIN) / 60) * timeValuePerHour); + const stopCost = stations.map((s) => (s.stopFee ?? 0) + ((s.detourMin + STOP_OVERHEAD_MIN) / 60) * timeValuePerHour); const minPerPct = stations.map((s) => s.minutesPerPct ?? 0); // Level consumed from stop i (-1 = origin) to stop j (n = destination). @@ -352,7 +355,7 @@ export function planRefuel(input: PlannerInput): PlanResult { id: s.id, km: s.km, litres, - cost: litres * s.price, + cost: litres * s.price + (s.stopFee ?? 0), detourMin: s.detourMin, chargeMin: (g - arrive) * minPerPct[j], arrivePct: arrive, From 319be9765a1c4a19782ded3c734fa7ba4cbba665 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:01:50 +0200 Subject: [PATCH 08/25] docs: OCPDB as Germany's default EV source, with prices Document the OCPDB scraper (merge, tariff choice, VAT, fees, feed health, cleanup), the ev_fees column, the new env vars and the EV price range. Pages that said chargers have no price now cover the priced German chargers: map, popup, route list and charging planner. --- docs/configuration/api-keys.md | 6 +- docs/configuration/countries-and-schedule.md | 25 +++-- docs/data/coverage.md | 15 +-- docs/data/ev-sources.md | 101 +++++++++++++++---- docs/data/how-scrapers-work.md | 22 ++-- docs/data/sources-at-a-glance.md | 3 +- docs/development.md | 4 +- docs/getting-started/first-start.md | 4 +- docs/index.md | 2 +- docs/operations/troubleshooting.md | 1 + docs/reference/api.md | 9 +- docs/reference/currencies.md | 9 +- docs/reference/data-model.md | 32 ++++-- docs/reference/environment-variables.md | 12 ++- docs/reference/fuel-types.md | 11 +- docs/reference/glossary.md | 18 ++-- docs/roadmap.md | 4 +- docs/using/ev-charging.md | 42 ++++---- docs/using/links.md | 2 +- docs/using/map.md | 2 +- 20 files changed, 214 insertions(+), 110 deletions(-) diff --git a/docs/configuration/api-keys.md b/docs/configuration/api-keys.md index ba9d76b..b376b57 100644 --- a/docs/configuration/api-keys.md +++ b/docs/configuration/api-keys.md @@ -8,11 +8,11 @@ This page lists every API key Pumperly can use: where to get it, what it unlocks |---|---|---|---| | [`TANKERKOENIG_API_KEY`](#tankerkonig-germany) | Tankerkönig | Germany's fuel prices | No German fuel prices. Each run fails and the last prices stay. | | [`FUELPRICES_DK_API_KEY`](#fuelpricesdk-denmark) | fuelprices.dk | Denmark's fuel prices | A fallback feed is tried, and it no longer answers. | -| [`PUMPERLY_OCM_API_KEY`](#open-charge-map) | Open Charge Map | EV chargers in every country without an official registry | No chargers from Open Charge Map. Germany still gets chargers from BNetzA. | +| [`PUMPERLY_OCM_API_KEY`](#open-charge-map) | Open Charge Map | EV chargers in every country without an official registry | No chargers from Open Charge Map. Germany still gets chargers from OCPDB. | | [`PUMPERLY_REVE_API_KEY`](#mapa-reve-spain) | Mapa REVE | Spain's chargers from the official registry | Spain's chargers come from Open Charge Map instead. | | [`NSW_FUEL_API_KEY`](#nsw-fuelcheck-australia) and [`NSW_FUEL_API_SECRET`](#nsw-fuelcheck-australia) | NSW FuelCheck | Fuel prices in New South Wales, Australia | Only Western Australia updates. | -Every other source, in every other country, is keyless. That includes Germany's charger registry, the [BNetzA Ladesäulenregister](../data/ev-sources.md). [Fuel price sources](../data/fuel-sources.md) and [EV charging sources](../data/ev-sources.md) list them all. +Every other source, in every other country, is keyless. That includes Germany's charger sources, the [Open ChargePoint Database](../data/ev-sources.md#ocpdb) and the [BNetzA Ladesäulenregister](../data/ev-sources.md#bnetza). [Fuel price sources](../data/fuel-sources.md) and [EV charging sources](../data/ev-sources.md) list them all. ## Setting a key {#setting-a-key} @@ -107,7 +107,7 @@ Keys are read when the app starts. Restart it after adding or changing one. - **Get a key:** create a free account at [openchargemap.org](https://openchargemap.org) and create an API key there. - **Variable:** [`PUMPERLY_OCM_API_KEY`](../reference/environment-variables.md#pumperly_ocm_api_key). - **How it is used:** the key goes in the `X-API-Key` header. Each `EV_` scraper runs once a day by default. A small country takes one request. The API returns at most 5,000 chargers per request, so for a large country the scraper splits the map into smaller boxes. By default it sends one request at a time across all countries, with 600 ms between requests, and at most 800 requests per country per run. [Open Charge Map tuning](../reference/environment-variables.md#open-charge-map-tuning) lists the settings. -- **Without it:** every Open Charge Map scraper skips its run and writes nothing. Chargers already in the database stay. Germany still gets chargers from the keyless BNetzA register, and Spain from REVE if you set that key. The log shows, for each country: +- **Without it:** every Open Charge Map scraper skips its run and writes nothing. Chargers already in the database stay. Germany still gets chargers from the keyless OCPDB, and Spain from REVE if you set that key. The log shows, for each country: ```text [ocm] PUMPERLY_OCM_API_KEY not set, skipping diff --git a/docs/configuration/countries-and-schedule.md b/docs/configuration/countries-and-schedule.md index 48c7ddb..37f177e 100644 --- a/docs/configuration/countries-and-schedule.md +++ b/docs/configuration/countries-and-schedule.md @@ -37,6 +37,7 @@ Each scheduled scraper has a key. You use keys in [`PUMPERLY_ENABLED_COUNTRIES`] | `EV_` | EV chargers for one country from Open Charge Map | `EV_FR`, `EV_US` | | `EV_ES_REVE` | Spain's chargers from Mapa REVE, the official registry | `EV_ES_REVE` | | `EV_DE_BNETZA` | Germany's chargers from the BNetzA Ladesäulenregister | `EV_DE_BNETZA` | +| `EV_DE_OCPDB` | Germany's chargers and their prices from the Open ChargePoint Database | `EV_DE_OCPDB` | | `STATIC_` | A community dataset committed to the repository | none ship today | `` is an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) country code. The 39 fuel-price countries are listed in [the country table](#default-country-and-fuel). The United States has chargers only, under `EV_US`, because it has no national fuel price source. @@ -68,7 +69,7 @@ With [`PUMPERLY_EV_ENABLED=0`](../reference/environment-variables.md#pumperly_ev | `ES,PT,FR` | Fuel prices and chargers for the three countries | | `AU,AU_NSW` | Fuel prices for Western Australia and New South Wales, and Australia's chargers | | `US` | Chargers in the United States only | -| `EV_DE` | Germany's chargers only, from the BNetzA register by default | +| `EV_DE` | Germany's chargers only, from OCPDB by default | | `ES,XX` | Same as `ES`. `XX` is ignored. | The value also sets the `enabledCountries` list that [`/api/config`](../reference/api.md) and `/api/stats` report. That list only holds the 39 fuel-price country codes, so `US`, `AU_NSW` and `EV_` entries never appear in it. @@ -86,32 +87,34 @@ Some sources are blocked or paused upstream, so a country can be enabled and sti ## Spain and Germany: one EV source each {#spain-and-germany-ev-sources} -Spain and Germany each have two possible charger sources: Open Charge Map, which is crowdsourced, and an official registry. The two overlap heavily, so running both would put two pins on most chargers. The scheduler always keeps exactly one per country. Listing both keys does not change that. +Spain and Germany each have more than one possible charger source: Open Charge Map, which is crowdsourced, and an official one. Germany has two official ones, the BNetzA register and OCPDB, which contains that register. The sources overlap heavily, so running two would put two pins on most chargers. The scheduler always keeps exactly one per country. Listing several keys does not change that. -| Country | Official registry | Picked when | Otherwise | +| Country | Official source | Picked when | Otherwise | |---|---|---|---| | Spain | Mapa REVE (`EV_ES_REVE`) | [`PUMPERLY_REVE_API_KEY`](../reference/environment-variables.md#pumperly_reve_api_key) is set | Open Charge Map (`EV_ES`) | -| Germany | BNetzA Ladesäulenregister (`EV_DE_BNETZA`) | Always, unless the next column applies | Open Charge Map (`EV_DE`) when [`PUMPERLY_DE_EV_SOURCE=ocm`](../reference/environment-variables.md#pumperly_de_ev_source) | +| Germany | Open ChargePoint Database (`EV_DE_OCPDB`) | Always, unless the next column applies | The BNetzA register (`EV_DE_BNETZA`) when [`PUMPERLY_DE_EV_SOURCE=bnetza`](../reference/environment-variables.md#pumperly_de_ev_source), Open Charge Map (`EV_DE`) when it is `ocm` | The log shows the choice at start-up: ```text [scraper] Spain EV: using Mapa REVE (official registry) instead of OpenChargeMap -[scraper] Germany EV: using the BNetzA Ladesäulenregister instead of OpenChargeMap +[scraper] Germany EV: using OCPDB (BNetzA register + operator feeds, with prices) ``` -Each registry deletes Open Charge Map's rows for its country once it has taken over: +Each official source deletes the other sources' rows for its country once it has taken over: -- **BNetzA** arrives as one daily file. Its first healthy run replaces the whole country and deletes the Open Charge Map rows. "Healthy" means it refreshed at least [`PUMPERLY_BNETZA_MIN_STATIONS`](../reference/environment-variables.md#pumperly_bnetza_min_stations) stations. +- **OCPDB** is read in full every 6 hours. Its first healthy run replaces the whole country and deletes the Open Charge Map and BNetzA rows. "Healthy" means it refreshed at least [`PUMPERLY_OCPDB_MIN_STATIONS`](../reference/environment-variables.md#pumperly_ocpdb_min_stations) stations. +- **BNetzA** arrives as one daily file. Its first healthy run replaces the whole country and deletes the Open Charge Map and OCPDB rows. "Healthy" means it refreshed at least [`PUMPERLY_BNETZA_MIN_STATIONS`](../reference/environment-variables.md#pumperly_bnetza_min_stations) stations. - **REVE** fills in over many hourly runs. Open Charge Map's Spanish rows stay on the map until REVE holds [`PUMPERLY_REVE_CUTOVER_RATIO`](../reference/environment-variables.md#pumperly_reve_cutover_ratio) of the registry, 95% by default. Until then, expect two pins on many Spanish chargers. [API keys](api-keys.md#mapa-reve-spain) has the timing. -!!! warning "Switching back leaves the registry's rows behind" - Nothing deletes a registry's rows if you switch back to Open Charge Map. Open Charge Map then adds its own, and most chargers get two pins. After switching Germany back to `ocm`, or removing the REVE key, delete the registry rows yourself: +!!! warning "Switching back leaves the official rows behind" + Open Charge Map deletes nothing when you switch back to it. It adds its own rows, and most chargers get two pins. After switching Germany to `ocm`, or removing the REVE key, delete the official rows yourself. Switching between `ocpdb` and `bnetza` needs nothing: each deletes the other's rows. ```sql -- Germany, after PUMPERLY_DE_EV_SOURCE=ocm DELETE FROM stations - WHERE country = 'DE' AND station_type = 'ev_charger' AND external_id LIKE 'bnetza-%'; + WHERE country = 'DE' AND station_type = 'ev_charger' + AND (external_id LIKE 'ocpdb-%' OR external_id LIKE 'bnetza-%'); -- Spain, after removing PUMPERLY_REVE_API_KEY DELETE FROM stations @@ -134,7 +137,7 @@ The defaults follow how often each source changes: | 1 hour | `FR`, `DE`, `EV_ES_REVE` | | 2 hours | `AT` | | 4 hours | `GB` | -| 6 hours | `SI`, `NL`, `BE`, `DK`, `NO`, `IS`, `CY` | +| 6 hours | `SI`, `NL`, `BE`, `DK`, `NO`, `IS`, `CY`, `EV_DE_OCPDB` | | 12 hours | every other fuel key, including `AU_NSW` | | 24 hours | every `EV_` key and `EV_DE_BNETZA` | diff --git a/docs/data/coverage.md b/docs/data/coverage.md index 739279a..21ac253 100644 --- a/docs/data/coverage.md +++ b/docs/data/coverage.md @@ -102,16 +102,17 @@ EV chargers come from three sources. Every fuel country above also gets EV charg |---|---|---|---|---| | Every supported country and the United States | `EV_`, for example `EV_FR` | [Open Charge Map](ev-sources.md#open-charge-map) | 24 h | `PUMPERLY_OCM_API_KEY` | | Spain | `EV_ES_REVE` | [Mapa REVE](ev-sources.md#mapa-reve) | 1 h | `PUMPERLY_REVE_API_KEY` | -| Germany | `EV_DE_BNETZA` | [BNetzA Ladesäulenregister](ev-sources.md#bnetza) | 24 h | nothing | +| Germany | `EV_DE_OCPDB` | [Open ChargePoint Database](ev-sources.md#ocpdb), with prices | 6 h | nothing | +| Germany, opt-in | `EV_DE_BNETZA` | [BNetzA Ladesäulenregister](ev-sources.md#bnetza) | 24 h | nothing | -Spain and Germany each use exactly one EV source at a time, never both: +Spain and Germany each use exactly one EV source at a time: - **Spain** uses Mapa REVE when `PUMPERLY_REVE_API_KEY` is set, and Open Charge Map otherwise. -- **Germany** uses the BNetzA register by default. Set `PUMPERLY_DE_EV_SOURCE=ocm` to use Open Charge Map instead. +- **Germany** uses OCPDB by default. Set `PUMPERLY_DE_EV_SOURCE=bnetza` for the BNetzA register, or `ocm` for Open Charge Map. -Without `PUMPERLY_OCM_API_KEY`, the Open Charge Map scrapers still run on schedule but write nothing. German chargers from BNetzA still load, because that source needs no key. +Without `PUMPERLY_OCM_API_KEY`, the Open Charge Map scrapers still run on schedule but write nothing. German chargers from OCPDB or BNetzA still load, because those sources need no key. -EV chargers have no prices. None of the three sources publishes a per-litre price, and per-kWh tariffs do not fit the price model. [EV charging sources](ev-sources.md) covers each source in detail. +Only German chargers from OCPDB have prices: the cheapest ad-hoc tariff per kWh, with any session and per-minute fees. Chargers from every other source have none. [EV charging sources](ev-sources.md) covers each source in detail. ## Community datasets {#community-datasets} @@ -119,12 +120,12 @@ Places with no source to scrape can be added as committed data files under [`src ## Choosing what runs {#choosing-what-runs} -By default every fuel scraper runs, including both Australian states. Each country also gets one EV source: Open Charge Map everywhere, including the United States, except that Germany uses the BNetzA register and Spain uses Mapa REVE when its key is set. These settings narrow that down: +By default every fuel scraper runs, including both Australian states. Each country also gets one EV source: Open Charge Map everywhere, including the United States, except that Germany uses OCPDB and Spain uses Mapa REVE when its key is set. These settings narrow that down: | Setting | Effect | |---|---| | `PUMPERLY_ENABLED_COUNTRIES` | Comma-separated country codes. Each code enables its fuel scraper and its EV scraper. `US` enables EV chargers for the United States. | -| `PUMPERLY_EV_ENABLED=0` | Turns off every EV scraper, including Mapa REVE and BNetzA. | +| `PUMPERLY_EV_ENABLED=0` | Turns off every EV scraper, including Mapa REVE, OCPDB and BNetzA. | | `PUMPERLY_SCRAPE_INTERVAL_` | Sets one scraper's interval in hours, using the key from the tables above. `0` stops that scraper. | | `PUMPERLY_SCRAPE_INTERVAL_HOURS` | Sets the interval for every scraper that has no per-key override. `0` stops all automatic scraping. | diff --git a/docs/data/ev-sources.md b/docs/data/ev-sources.md index 65ce3d2..0bc7cf6 100644 --- a/docs/data/ev-sources.md +++ b/docs/data/ev-sources.md @@ -1,34 +1,36 @@ # EV charging sources -This page describes where Pumperly's EV charging locations come from, how each source is read and which settings tune it. EV chargers carry a location, an operator and an address. They carry no price. +This page describes where Pumperly's EV charging locations come from, how each source is read and which settings tune it. EV chargers carry a location, an operator and an address. German chargers from OCPDB also carry a price per kWh and any fees on top. Every other charger has no price. -## The three sources {#the-three-sources} +## The sources {#the-three-sources} | Source | Covers | Key | Every | Licence | |---|---|---|---|---| | [Open Charge Map](#open-charge-map) | Every supported country and the United States | `PUMPERLY_OCM_API_KEY` | 24 h | Open Database License (ODbL) | | [Mapa REVE](#mapa-reve) | Spain | `PUMPERLY_REVE_API_KEY` | 1 h | Non-commercial use, credit Red Eléctrica de España | +| [Open ChargePoint Database (OCPDB)](#ocpdb) | Germany, with prices | None | 6 h | CC BY 4.0 for the register and EnBW, CC0 for the other feeds | | [BNetzA Ladesäulenregister](#bnetza) | Germany | None | 24 h | CC BY 4.0, credit "Bundesnetzagentur.de" | -Open Charge Map is crowdsourced: anyone can add or edit a charger. Mapa REVE and the BNetzA register are official registries that charge point operators must file into, so they are authoritative where Open Charge Map is crowdsourced. +Open Charge Map is crowdsourced: anyone can add or edit a charger. Mapa REVE and the BNetzA register are official registries that charge point operators must file into, so they are authoritative where Open Charge Map is crowdsourced. OCPDB contains the BNetzA register and adds the operators' own feeds, which carry ad-hoc tariffs. -None of the three publishes a per-litre price. Mapa REVE publishes connector tariffs, but they are per kWh and per session, which the price model cannot hold, so they are left out. +Only OCPDB's prices are stored. Mapa REVE also publishes connector tariffs, but its scraper leaves them out. ## Which source a country uses {#which-source} -Every other country uses Open Charge Map. Spain and Germany each have a second, official source, and each uses exactly one of its two sources at a time. Running both would put two pins on almost every charger, because the sources overlap heavily. +Every other country uses Open Charge Map. Spain and Germany also have official sources, and each country uses exactly one source at a time. Running two would put two pins on almost every charger, because the sources overlap heavily. ```mermaid flowchart TD ES[Spain] --> K{PUMPERLY_REVE_API_KEY set?} K -- Yes --> REVE[Mapa REVE, EV_ES_REVE] K -- No --> OCMES[Open Charge Map, EV_ES] - DE[Germany] --> S{PUMPERLY_DE_EV_SOURCE=ocm?} - S -- Yes --> OCMDE[Open Charge Map, EV_DE] - S -- No --> BNETZA[BNetzA register, EV_DE_BNETZA] + DE[Germany] --> S{PUMPERLY_DE_EV_SOURCE} + S -- ocm --> OCMDE[Open Charge Map, EV_DE] + S -- bnetza --> BNETZA[BNetzA register, EV_DE_BNETZA] + S -- anything else --> OCPDB[OCPDB, EV_DE_OCPDB] ``` -The scheduler and the manual scraper command apply the same rule, from [`spain-ev-source.ts`](https://github.com/GeiserX/Pumperly/blob/main/src/scrapers/spain-ev-source.ts) and [`germany-ev-source.ts`](https://github.com/GeiserX/Pumperly/blob/main/src/scrapers/germany-ev-source.ts). In the scheduler, `PUMPERLY_EV_ENABLED=0` turns off all three sources at once. +The scheduler and the manual scraper command apply the same rule, from [`spain-ev-source.ts`](https://github.com/GeiserX/Pumperly/blob/main/src/scrapers/spain-ev-source.ts) and [`germany-ev-source.ts`](https://github.com/GeiserX/Pumperly/blob/main/src/scrapers/germany-ev-source.ts). In the scheduler, `PUMPERLY_EV_ENABLED=0` turns off every EV source at once. !!! warning "Naming a source by hand overrides the rule" The manual command `npm run scraper:run -- --country=all` picks one source per country, like the scheduler. Naming a scraper explicitly, such as `--country=EV_ES`, runs it even when the other source is active. One such run puts back every Open Charge Map row the registry had retired. They stay, as duplicates, until the registry's next error-free run deletes them again. @@ -56,7 +58,7 @@ Only chargers marked operational are requested. Each one becomes a station with: ### Without an API key {#ocm-without-a-key} -The scrapers still run on schedule. Each one logs `PUMPERLY_OCM_API_KEY not set, skipping` and writes nothing. Because the run returned no stations, the scheduler reports it with one error. German chargers still load from the BNetzA register, which needs no key. +The scrapers still run on schedule. Each one logs `PUMPERLY_OCM_API_KEY not set, skipping` and writes nothing. Because the run returned no stations, the scheduler reports it with one error. German chargers still load from OCPDB or the BNetzA register, which need no key. ### How a large country is fetched {#ocm-tiling} @@ -201,6 +203,64 @@ The workflow only tracks the project's own key. If you run your own instance, ke --8<-- ".github/workflows/reve-key-renewal.yml" ``` +## Open ChargePoint Database (OCPDB), Germany {#ocpdb} + +The [Open ChargePoint Database](https://api.mobidata-bw.de/ocpdb/) is run by MobiData BW for the state of Baden-Württemberg, but covers all of Germany. It merges the BNetzA register with the operators' own DATEX II feeds from Mobilithek: EnBW, EcoMovement, Chargecloud, Wirelane, e-clearing.net, Tesla and others. Those feeds carry ad-hoc tariffs, the price a driver pays without a contract, for about 97,000 charge points. It is Germany's default source. The code is in [`src/scrapers/ocpdb.ts`](https://github.com/GeiserX/Pumperly/blob/main/src/scrapers/ocpdb.ts). + +| | | +|---|---| +| Source key | `ocpdb` | +| Scraper key | `EV_DE_OCPDB` | +| Station ids | `ocpdb-_` | +| Every | 6 h | +| API key | None | +| Licence | The register and EnBW are CC BY 4.0, the other feeds CC0. The attribution is in the app's legal notice. | + +### How the data is read {#ocpdb-reading} + +Each run reads four endpoints in parallel: locations, tariffs, the links from tariffs to charge points, and the health of each operator feed. The OCPI endpoints are paged, 1,000 items per request. Each page, and the feed-health request, is retried once. + +- **Only German locations with a live charge point are kept.** Locations outside a padded bounding box around Germany, and charge points marked removed or planned, are skipped and counted. +- **Locations within 25 m merge into one pin.** A charger reported by its operator is often in the register too, under another operator name and a few metres away. When the register lists the charger, its location anchors the pin, so the coordinates are the official ones. The station id is built from the anchor's coordinates. +- **The brand is the operator,** and the name joins the operator and the site. When a merged location supplies the pin's price, its operator becomes the brand, so a pin never shows one operator's name next to another's price. + +Each run logs what it read: + +```text +[ocpdb] DE: tariffs → priced EVSEs; locations → stations, priced (merged , skipped without live EVSE, outside DE, unparseable) +``` + +### How a price is chosen {#ocpdb-prices} + +A charger can have several tariffs. The pin shows one: + +1. **Only ad-hoc tariffs count.** Tariffs for contract customers, and links with no audience, are skipped. +2. **Only tariffs from healthy feeds count.** A feed that OCPDB marks as failed, or that has not refreshed in [`PUMPERLY_OCPDB_MAX_FEED_AGE_DAYS`](../reference/environment-variables.md#pumperly_ocpdb_max_feed_age_days) days, has its tariffs ignored, and its chargers show unpriced. The price's `reported_at` is when its feed last refreshed. +3. **Prices are gross.** Ad-hoc charging is a consumer price, and German law requires it shown with VAT. A price marked net gets the VAT rate it lists, or 19 % when it lists none. A price with no tax data is read as gross. +4. **Prices outside EUR 0.10 to 2.00 per kWh are dropped** before the cheapest is picked, so a bogus 0.00 cannot hide a real price. +5. **The cheapest wins,** with the session fee spread over a 30 kWh session. A low rate cannot hide a large fee. + +The fees on top of the per-kWh price are stored in [`ev_fees`](../reference/data-model.md#ev_fees): + +- **Session fee:** fixed fees that always apply are added up. When the fee depends on the time of day, the weekday or other conditions, every option is kept and the popup lists them. The highest is the one used to rank tariffs and to cost a planned stop. +- **Per-minute fee:** the earliest-starting time-based charge that costs anything, with the minute it starts. These are blocking fees, typically 30 minutes to 4 hours into the session. OCPI defines them per hour, but every feed seen uses them per minute, so Pumperly reads them per minute. + +### When the feed health is unavailable {#ocpdb-feed-health} + +Without the feed-health endpoint, no tariff can be vouched for, and a run without prices would delete the current ones. So when that request fails twice, the whole run fails and writes nothing. The stations and prices from the last good run stay until the next run. + +### Cleanup after each run {#ocpdb-cleanup} + +Cleanup works as for [BNetzA](#bnetza-cleanup). After a run with no errors, if the run refreshed at least [`PUMPERLY_OCPDB_MIN_STATIONS`](../reference/environment-variables.md#pumperly_ocpdb_min_stations) stations, the scraper deletes: + +1. **OCPDB stations the run did not refresh.** These left the database, or their anchor's coordinates changed, which changes the id. +2. **Every German Open Charge Map and BNetzA charger.** Once OCPDB has landed, those rows are duplicates. + +| Variable | Default | Effect | +|---|---|---| +| `PUMPERLY_OCPDB_MIN_STATIONS` | `10000` | Stations a run must refresh before either deletion runs. A value below 1 or not a number falls back to the default. | +| `PUMPERLY_OCPDB_MAX_FEED_AGE_DAYS` | `3` | Days a feed may go without refreshing before its tariffs are ignored. A value of 0 or less, or not a number, falls back to the default. | + ## BNetzA Ladesäulenregister, Germany {#bnetza} The Ladesäulenregister is the German charging point register kept by the Bundesnetzagentur, the federal network agency. Every operator of a publicly accessible charge point must file into it under §5 of the Ladesäulenverordnung, the charging station ordinance. It holds about 74,000 locations. The code is in [`src/scrapers/bnetza.ts`](https://github.com/GeiserX/Pumperly/blob/main/src/scrapers/bnetza.ts). @@ -214,14 +274,15 @@ The Ladesäulenregister is the German charging point register kept by the Bundes | API key | None | | Licence | CC BY 4.0, attribution "Bundesnetzagentur.de" | -The register only lists operators who completed the notification procedure and agreed to publication, so it is not complete. It is still several times larger than Open Charge Map's German coverage. +The register only lists operators who completed the notification procedure and agreed to publication, so it is not complete. It is still several times larger than Open Charge Map's German coverage. It has no prices: [OCPDB](#ocpdb) contains the same register and adds them. ### Choosing Germany's source {#bnetza-source-choice} -| `PUMPERLY_DE_EV_SOURCE` | Germany's EV source | +| `PUMPERLY_DE_EV_SOURCE` (any case) | Germany's EV source | |---|---| -| unset, or any value other than `ocm` | BNetzA register | -| `ocm` (any case) | Open Charge Map | +| unset, `ocpdb`, or any value not listed here | OCPDB | +| `bnetza` | BNetzA register | +| `ocm` | Open Charge Map | ### How the file is read {#bnetza-reading} @@ -244,19 +305,19 @@ Each run logs how many rows it read, merged and skipped: The file arrives complete in one run, so Germany needs no backfill period. After a run with no errors, the scraper counts the BNetzA stations that run refreshed. If the count reaches the safety floor, it makes two deletions: 1. **BNetzA stations the run did not refresh.** These left the register, or their operator corrected the coordinates, which changes the id. -2. **Every German Open Charge Map charger.** Once BNetzA has landed, those rows are duplicates. +2. **Every German Open Charge Map and OCPDB charger.** Once BNetzA has landed, those rows are duplicates. | Variable | Default | Effect | |---|---|---| | `PUMPERLY_BNETZA_MIN_STATIONS` | `10000` | Stations a run must refresh before either deletion runs. A value below 1 or not a number falls back to the default. | -The floor counts only rows this run wrote, not rows already in the database. A broken download that refreshes a handful of stations can therefore never delete the rest, and never retires Open Charge Map on its strength. +The floor counts only rows this run wrote, not rows already in the database. A broken download that refreshes a handful of stations can therefore never delete the rest, and never retires the other sources on its strength. ## Switching a source back {#switching-back} -The handovers only run in one direction. Registry runs delete Open Charge Map rows, but nothing deletes registry rows once their scraper stops running. If you switch Spain or Germany back to Open Charge Map, the registry's pins stay, and Open Charge Map adds its own next to them. +Open Charge Map deletes nothing when it takes over. Official sources delete Open Charge Map's rows, and OCPDB and BNetzA delete each other's, but nothing deletes the official rows once their scraper stops running and Open Charge Map runs instead. If you switch Spain or Germany back to Open Charge Map, the official pins stay, and Open Charge Map adds its own next to them. -To clear the old registry rows after switching back, delete them in `psql` against the Pumperly database: +To clear the old rows after switching back, delete them in `psql` against the Pumperly database: === "Spain, after removing the REVE key" @@ -273,7 +334,7 @@ To clear the old registry rows after switching back, delete them in `psql` again DELETE FROM stations WHERE country = 'DE' AND station_type = 'ev_charger' - AND external_id LIKE 'bnetza-%'; + AND (external_id LIKE 'ocpdb-%' OR external_id LIKE 'bnetza-%'); ``` -EV chargers have no price rows, so nothing else needs deleting. [Backing up the database](../operations/backup-and-restore.md) explains how to take a backup first. +A charger's prices are deleted with it, so nothing else needs deleting. [Backing up the database](../operations/backup-and-restore.md) explains how to take a backup first. diff --git a/docs/data/how-scrapers-work.md b/docs/data/how-scrapers-work.md index ed69236..183eba6 100644 --- a/docs/data/how-scrapers-work.md +++ b/docs/data/how-scrapers-work.md @@ -19,11 +19,11 @@ A scraper declares two names and implements one method. The base class does ever | Record | Fields | | --- | --- | | `RawStation` | `externalId`, `name`, `brand` (or `null`), `address`, `city`, `province` (or `null`), `latitude`, `longitude`, `stationType` (`"fuel"`, `"ev_charger"` or `"both"`) | -| `RawFuelPrice` | `stationExternalId` (matches a station's `externalId` in the same result), `fuelType` (one of the [fuel type codes](../reference/fuel-types.md)), `price` (per litre, in local currency), `currency` (ISO 4217) | +| `RawFuelPrice` | `stationExternalId` (matches a station's `externalId` in the same result), `fuelType` (one of the [fuel type codes](../reference/fuel-types.md)), `price` (per litre, in local currency; per kWh for `EV`), `currency` (ISO 4217). Optional: `evFees` (`EV` only, written to [`ev_fees`](../reference/data-model.md#ev_fees)) and `reportedAt` (when the upstream last vouched for the price) | The `externalId` is the source's own id for the station, usually with a short prefix such as `ocm-` or `se-bp-`. Together with `country` it identifies a station across runs. [Data model](../reference/data-model.md) explains both columns in full. -EV charger scrapers return stations with `stationType: "ev_charger"` and an empty price list. +EV charger scrapers return stations with `stationType: "ev_charger"`. Most return an empty price list. The OCPDB scraper returns one `EV` price per priced charger. `fetch()` should throw when the upstream answers with an HTTP error or with data of the wrong shape. The base class turns the throw into a failed run that changes nothing. A single malformed row is different: skip it and keep going, so one bad record cannot cost the whole country a run. @@ -71,6 +71,7 @@ Which band applies depends on the fuel and the currency: | Case | Accepted range | | --- | --- | +| `EV`, per kWh | `EUR` only, from 0.10 to 2.00. A price in any other currency is dropped. | | `H2`, `CNG`, `LNG` and `ADBLUE`, any currency | At least 0.05 and below 100 | | `EUR`, `GBP` and `CHF` | From `PUMPERLY_PRICE_MIN` to `PUMPERLY_PRICE_MAX`. The defaults are 0.30 and 4.00. | | Any other currency with a band | The band for that currency (table below) | @@ -92,7 +93,7 @@ The per-currency bands live in the `PRICE_BANDS` table in [`base.ts`](https://gi The table also holds bands for EUR, GBP and CHF, but a run uses the two environment variables for those three currencies instead. If `PUMPERLY_PRICE_MIN` or `PUMPERLY_PRICE_MAX` is not a number, the default is used, so a typo cannot switch the check off. -After the price filter, a station of type `fuel` with no valid price left is dropped from this run. EV chargers are kept even though they carry no prices. +After the price filter, a station of type `fuel` with no valid price left is dropped from this run. EV chargers are kept, priced or not. ### 2. The empty-fetch guard @@ -116,7 +117,7 @@ The run looks up the database id of every station in the country, then maps each Then one transaction replaces the prices: 1. Delete every price row whose `source` is this scraper's source and whose station is in this scraper's country. -2. Insert the new prices in batches of 2,000. Each row gets `reported_at` set to the current time. +2. Insert the new prices in batches of 2,000. Each row gets `reported_at` set to the current time, unless the scraper passed its own `reportedAt`. Because it is a single transaction, readers see either the old set or the new set, never a mix. The transaction may run for up to 120 seconds. If it fails, it rolls back and the old prices stay. @@ -140,7 +141,7 @@ This is how a station disappears from the map when its source stops listing it. The cleanup runs only when every station batch in step 3 succeeded. If a batch failed, the stations in it may not have been written this run, and their prices were already replaced in step 4. Deleting orphans then could remove stations that are only missing because of the failure. So the run records `Skipped orphan cleanup: one or more station batches failed` and leaves them alone. !!! note "EV chargers are never cleaned up by the base class" - The cleanup only deletes rows with `station_type = 'fuel'`. An EV charger that its source stops listing stays in the table. A scraper that needs to remove EV rows does it in its own `run()` override, as the Mapa REVE and BNetzA scrapers do. See [EV charging sources](ev-sources.md). + The cleanup only deletes rows with `station_type = 'fuel'`. An EV charger that its source stops listing stays in the table. A scraper that needs to remove EV rows does it in its own `run()` override, as the Mapa REVE, BNetzA and OCPDB scrapers do. The German ones share `BaseScraper.sweepEvRows()`. See [EV charging sources](ev-sources.md). ### The result @@ -178,7 +179,7 @@ Every scraper has a **key** in the scheduler. The key is what you use in environ | `XX` | The fuel price scraper for country `XX` | `SE` | | `XX_NAME` | A second fuel scraper for the same country | `AU_NSW` (New South Wales, next to `AU` for Western Australia) | | `EV_XX` | The OpenChargeMap EV scraper for country `XX` | `EV_SE` | -| `EV_XX_NAME` | An official EV registry that replaces OpenChargeMap for `XX` | `EV_ES_REVE`, `EV_DE_BNETZA` | +| `EV_XX_NAME` | An official EV source that replaces OpenChargeMap for `XX` | `EV_ES_REVE`, `EV_DE_BNETZA`, `EV_DE_OCPDB` | | `STATIC_NAME` | A committed community dataset (see [`src/scrapers/data/README.md`](https://github.com/GeiserX/Pumperly/blob/main/src/scrapers/data/README.md)) | none ship today | ### Which scrapers run @@ -187,7 +188,7 @@ Every scraper has a **key** in the scheduler. The key is what you use in environ 2. If `PUMPERLY_ENABLED_COUNTRIES` is unset, every key is selected. 3. If it is set, only the keys it lists are selected. For each listed code that is not itself an `EV_` key, `EV_` is added when such a key exists. So `SE` selects `SE` and `EV_SE`, and `US` selects `EV_US` alone, because the United States has EV coverage only. 4. `PUMPERLY_EV_ENABLED=0` removes every `EV_` key. -5. Spain and Germany each keep exactly one EV source. Spain runs `EV_ES_REVE` when `PUMPERLY_REVE_API_KEY` is set and `EV_ES` otherwise. Germany runs `EV_DE_BNETZA` unless `PUMPERLY_DE_EV_SOURCE=ocm`. Running both would put two pins on most chargers. +5. Spain and Germany each keep exactly one EV source. Spain runs `EV_ES_REVE` when `PUMPERLY_REVE_API_KEY` is set and `EV_ES` otherwise. Germany runs `EV_DE_OCPDB` unless `PUMPERLY_DE_EV_SOURCE` is `bnetza` (`EV_DE_BNETZA`) or `ocm` (`EV_DE`). Running two would put two pins on most chargers. !!! note "A second scraper for a country needs its own key" Listing `AU` in `PUMPERLY_ENABLED_COUNTRIES` selects `AU` and `EV_AU`, not `AU_NSW`. List `AU_NSW` too if you set the variable and want New South Wales. @@ -271,12 +272,13 @@ async run(): Promise { After the first successful handover the retired sources have no rows left, so later runs delete nothing. The step is safe to leave in place. -The two EV registries use variations of the same idea, because EV rows are never removed by the base class: +The EV scrapers use variations of the same idea, because EV rows are never removed by the base class: | Scraper | Retires | Condition | | --- | --- | --- | | Sweden, `bensinpriser` | Prices of the retired sources, then Swedish fuel stations left without a price | Error-free run and at least 200 stations | | Mapa REVE, `EV_ES_REVE` | Spanish OpenChargeMap chargers (`ocm-` ids) | Error-free run and the local REVE rows reach a share of the registry, 95% by default (`PUMPERLY_REVE_CUTOVER_RATIO`) | -| BNetzA, `EV_DE_BNETZA` | German OpenChargeMap chargers, and its own chargers that left the register | Error-free run and at least `PUMPERLY_BNETZA_MIN_STATIONS` rows (default 10,000) refreshed by this run | +| BNetzA, `EV_DE_BNETZA` | German OpenChargeMap and OCPDB chargers, and its own chargers that left the register | Error-free run and at least `PUMPERLY_BNETZA_MIN_STATIONS` rows (default 10,000) refreshed by this run | +| OCPDB, `EV_DE_OCPDB` | German OpenChargeMap and BNetzA chargers, and its own chargers that left the database | Error-free run and at least `PUMPERLY_OCPDB_MIN_STATIONS` rows (default 10,000) refreshed by this run | -[EV charging sources](ev-sources.md) describes the two EV handovers in detail. [Adding a country](adding-a-country.md) covers when a new scraper needs this pattern. +[EV charging sources](ev-sources.md) describes the EV handovers in detail. [Adding a country](adding-a-country.md) covers when a new scraper needs this pattern. diff --git a/docs/data/sources-at-a-glance.md b/docs/data/sources-at-a-glance.md index 737992e..eaee61b 100644 --- a/docs/data/sources-at-a-glance.md +++ b/docs/data/sources-at-a-glance.md @@ -42,11 +42,12 @@ One table per kind of source. The pages under Data sources explain each source i |---|---|---| | [Open Charge Map](https://openchargemap.org) | All supported countries + United States (EV-only) | ODbL | | [Mapa REVE](https://www.mapareve.es) (Red Eléctrica de España) | Spain — official operator-reported registry | Non-commercial, attribution required | +| [Open ChargePoint Database](https://api.mobidata-bw.de/ocpdb/) (MobiData BW) | Germany — the Ladesäulenregister merged with operator feeds, with ad-hoc prices | CC BY 4.0 for the register and EnBW, CC0 for the other feeds | | [Ladesäulenregister](https://www.bundesnetzagentur.de/DE/Fachthemen/ElektrizitaetundGas/E-Mobilitaet/Ladesaeulenkarte/start.html) (Bundesnetzagentur) | Germany — official operator-reported registry, ~74,000 locations | CC BY 4.0, attribution "Bundesnetzagentur.de" | Spain uses Mapa REVE when `PUMPERLY_REVE_API_KEY` is set: it is the registry every Spanish charge point operator files into, so it is authoritative where Open Charge Map is crowdsourced. The two overlap heavily, so Open Charge Map stops being scraped for Spain immediately, its existing Spanish rows stay visible while REVE backfills, and they are deleted once REVE reaches 95% of the registry. Expect duplicate Spanish pins until then. -Germany uses the BNetzA Ladesäulenregister by default — no API key, no signup. Every operator of a publicly accessible charge point must file into it under §5 Ladesäulenverordnung, so like REVE it is authoritative where Open Charge Map is crowdsourced, and it is several times larger. It arrives as one daily bulk file, so there is no backfill period: Open Charge Map stops being scraped for Germany, and its existing German rows are retired on the first healthy run. Set `PUMPERLY_DE_EV_SOURCE=ocm` to keep Open Charge Map instead. +Germany uses the Open ChargePoint Database (OCPDB) by default — no API key, no signup. It contains the BNetzA Ladesäulenregister, which every operator of a publicly accessible charge point must file into under §5 Ladesäulenverordnung, and merges it with the operators' own feeds, which carry ad-hoc prices. It is read in full every 6 hours, so there is no backfill period: Open Charge Map and BNetzA stop being scraped for Germany, and their existing German rows are retired on the first healthy run. Set `PUMPERLY_DE_EV_SOURCE=bnetza` to use the register alone, without prices, or `ocm` to keep Open Charge Map. ## Map & routing diff --git a/docs/development.md b/docs/development.md index 2f53d5e..89d05f0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -346,7 +346,7 @@ The command line in [`src/scrapers/cli.ts`](https://github.com/GeiserX/Pumperly/ ```bash npm run scraper:run -- --country=FR npm run scraper:run -- --country=ES,PT --once -npm run scraper:run -- --country=EV_DE_BNETZA +npm run scraper:run -- --country=EV_DE_OCPDB npm run scraper:run -- --country=all ``` @@ -359,7 +359,7 @@ The codes are the scheduler's codes: - A country code such as `ES` runs that country's fuel scraper. `AU` runs both Australian sources, Western Australia and New South Wales. - `EV_XX` runs Open Charge Map for country `XX`, for example `EV_FR` or `EV_US`. -- `EV_ES_REVE` and `EV_DE_BNETZA` run the Spanish and German official charger registries. +- `EV_ES_REVE` runs the Spanish official charger registry. `EV_DE_OCPDB` and `EV_DE_BNETZA` run the two German official sources. - `all` runs every scraper. For Spain and Germany it picks one EV source, the same way the app does. Naming `EV_ES` or `EV_DE` yourself runs Open Charge Map for that country regardless. An unknown code prints the list of valid ones and exits. diff --git a/docs/getting-started/first-start.md b/docs/getting-started/first-start.md index 6b2523a..dbb9c68 100644 --- a/docs/getting-started/first-start.md +++ b/docs/getting-started/first-start.md @@ -58,10 +58,10 @@ When the Spanish or German EV source is an official registry, the app logs it at ```text [scraper] Spain EV: using Mapa REVE (official registry) instead of OpenChargeMap -[scraper] Germany EV: using the BNetzA Ladesäulenregister instead of OpenChargeMap +[scraper] Germany EV: using OCPDB (BNetzA register + operator feeds, with prices) ``` -Each line appears only when that country's EV scraper is enabled. The Spanish line also needs `PUMPERLY_REVE_API_KEY` to be set. The German line appears unless `PUMPERLY_DE_EV_SOURCE=ocm`. See [EV charging sources](../data/ev-sources.md). +Each line appears only when that country's EV scraper is enabled. The Spanish line also needs `PUMPERLY_REVE_API_KEY` to be set. The German line above appears unless `PUMPERLY_DE_EV_SOURCE` is `bnetza` or `ocm`. With `bnetza` it reads `using the BNetzA Ladesäulenregister instead of OpenChargeMap`. See [EV charging sources](../data/ev-sources.md). !!! note "Australia is two scrapers" `AU` runs the Western Australia source. New South Wales is a separate scraper, `AU_NSW`. List it too if you set `PUMPERLY_ENABLED_COUNTRIES`, for example `AU,AU_NSW`. It needs `NSW_FUEL_API_KEY` and `NSW_FUEL_API_SECRET`. See [Countries and scrape schedule](../configuration/countries-and-schedule.md). diff --git a/docs/index.md b/docs/index.md index 35ca73b..b110f84 100644 --- a/docs/index.md +++ b/docs/index.md @@ -134,7 +134,7 @@ flowchart LR ## What it does not do -- It has no prices for EV charging. The sources publish charger locations, not tariffs. +- It has EV charging prices for Germany only. Elsewhere the sources publish charger locations, not tariffs. - It has no fuel prices for the United States, only chargers. No national price feed exists. - It does not route without Valhalla, and its search box finds nothing without Photon. The shipped compose file starts neither. - It does not store prices for a paused or blocked source. The last good prices stay on the map, and each popup says how old they are. diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index 0c5de96..e7d9085 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -162,6 +162,7 @@ The ranges, from [`src/scrapers/base.ts`](https://github.com/GeiserX/Pumperly/bl | Prices | Accepted range | |--------|----------------| | Any price of 0 or less | Always rejected | +| `EV`, per kWh | `EUR` only, from 0.10 to 2.00 | | `H2`, `CNG`, `LNG` and `ADBLUE`, in any currency | At least 0.05 and below 100 | | EUR, GBP and CHF | From `PUMPERLY_PRICE_MIN` to `PUMPERLY_PRICE_MAX`, by default 0.30 to 4.00 | | Other currencies | A fixed band per currency, listed below | diff --git a/docs/reference/api.md b/docs/reference/api.md index b3a3fc8..54246b4 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -165,7 +165,7 @@ Returns the stations inside a rectangle that sell one fuel, each with its latest **What it selects:** - For a fuel other than `EV`, a station is returned only if it has at least one price for that fuel. The price shown is the newest one by `reported_at`. -- For `EV`, it returns every station whose type is `ev_charger` or `both`. These features have no `price` and no `reportedAt`. Their `currency` is always `EUR` and means nothing. +- For `EV`, it returns every station whose type is `ev_charger` or `both`, priced or not. A charger with an `EV` price row carries its newest `price` per kWh, its `currency`, `reportedAt` and any `evFees`. Unpriced chargers have no `price` and no `reportedAt`, and their `currency` is `EUR` and means nothing. **Limit:** at most 20,000 features. The query has no `ORDER BY`, so a box with more stations returns an arbitrary 20,000. Nothing in the response says it was cut. The server logs `[stations] result truncated at 20000`. Ask for a smaller box if you need them all. @@ -449,10 +449,11 @@ curl -sN -X POST http://localhost:3000/api/route-detour \ | `brand` | yes, may be `null` | Brand or operator. | | `address`, `city` | yes | As the source gives them. | | `fuelType` | yes | The fuel code you asked for. | -| `currency` | yes | ISO 4217 code of `price`. `EUR` on EV chargers, where it carries no meaning. | -| `price` | no | Latest price for `fuelType`, in `currency`. Absent on EV chargers. | -| `reportedAt` | no | ISO 8601 time the price was stored. This is when Pumperly's scraper wrote it, not a time the source published. Absent on EV chargers. | +| `currency` | yes | ISO 4217 code of `price`. `EUR` on unpriced EV chargers, where it carries no meaning. | +| `price` | no | Latest price for `fuelType`, in `currency`. Per kWh for `EV`. Absent on unpriced EV chargers. | +| `reportedAt` | no | ISO 8601 time the price was stored. This is when Pumperly's scraper wrote it, not a time the source published. For OCPDB prices it is when the operator feed last refreshed. Absent on unpriced EV chargers. | | `powerKw` | no | EV chargers only: highest single-connector power in kW. Absent when the source publishes none. | +| `evFees` | no | EV chargers only: fees on top of `price`, in the price's currency. Shaped like the [`ev_fees`](data-model.md#ev_fees) column. Absent when there are none. | | `distanceKm` | only `/api/stations/nearest` | Approximate distance from the query point. | | `routeFraction` | only `/api/route-stations` | Position along the route, `0` to `1`. | diff --git a/docs/reference/currencies.md b/docs/reference/currencies.md index 914988c..0b08250 100644 --- a/docs/reference/currencies.md +++ b/docs/reference/currencies.md @@ -181,10 +181,11 @@ A price band is the range of prices Pumperly accepts for one currency. It catche The rules live in `BaseScraper.run()` in [`src/scrapers/base.ts`](https://github.com/GeiserX/Pumperly/blob/main/src/scrapers/base.ts). For each price, in this order: 1. A price of zero or less is dropped. -2. `CNG`, `LNG`, `H2` and `ADBLUE` are kept if the price is at least 0.05 and below 100, whatever the currency. The rest of this list does not apply to them. -3. A price in `EUR`, `GBP` or `CHF` is kept if it is from [`PUMPERLY_PRICE_MIN`](environment-variables.md#pumperly_price_min) to [`PUMPERLY_PRICE_MAX`](environment-variables.md#pumperly_price_max). The defaults are 0.30 and 4.00. The same two numbers apply to all three currencies. -4. Any other currency is checked against its row in the table below. Both ends are inclusive. -5. A currency with no row is checked against a wide default of 0.1 to 100,000. +2. `EV` prices are per kWh. They are kept only in `EUR`, from 0.10 to 2.00. The rest of this list does not apply to them. +3. `CNG`, `LNG`, `H2` and `ADBLUE` are kept if the price is at least 0.05 and below 100, whatever the currency. The rest of this list does not apply to them. +4. A price in `EUR`, `GBP` or `CHF` is kept if it is from [`PUMPERLY_PRICE_MIN`](environment-variables.md#pumperly_price_min) to [`PUMPERLY_PRICE_MAX`](environment-variables.md#pumperly_price_max). The defaults are 0.30 and 4.00. The same two numbers apply to all three currencies. +5. Any other currency is checked against its row in the table below. Both ends are inclusive. +6. A currency with no row is checked against a wide default of 0.1 to 100,000. | Currency | Minimum | Maximum | |---|---|---| diff --git a/docs/reference/data-model.md b/docs/reference/data-model.md index f25bda8..f079004 100644 --- a/docs/reference/data-model.md +++ b/docs/reference/data-model.md @@ -33,6 +33,7 @@ erDiagram varchar currency timestamptz reported_at varchar source + jsonb ev_fees } ``` @@ -88,6 +89,12 @@ The shipped files, included here word for word: --8<-- "prisma/migrations/20260930000000_station_max_power_kw/migration.sql" ``` +=== "EV fees migration" + + ```sql + --8<-- "prisma/migrations/20261022000000_fuel_price_ev_fees/migration.sql" + ``` + ## `stations` One row per place: a fuel station, an EV charger, or a site that is both. @@ -104,7 +111,7 @@ One row per place: a fuel station, an EV charger, or a site that is both. | `province` | `text` | Yes | Region, state or county, when the source gives one. | | `station_type` | `varchar(20)` | No | `fuel`, `ev_charger` or `both`. Defaults to `fuel`. | | `geom` | `geometry(Point, 4326)` | Yes | The position, as a PostGIS point in WGS84 longitude and latitude. Scrapers always set it. | -| `max_power_kw` | `smallint` | Yes | EV chargers: highest single-connector power in kW, set by the BNetzA, OCM and REVE scrapers. `null` for fuel stations and chargers with no published power. | +| `max_power_kw` | `smallint` | Yes | EV chargers: highest single-connector power in kW, set by the OCPDB, BNetzA, OCM and REVE scrapers. `null` for fuel stations and chargers with no published power. | | `created_at` | `timestamptz` | No | When the row was first inserted. Never changed afterwards. | | `updated_at` | `timestamptz` | No | When a scraper run last wrote this station. Every successful upsert sets it to the current time, even when nothing changed. | @@ -113,7 +120,7 @@ One row per place: a fuel station, an EV charger, or a site that is both. | Value | Written by | Shown for | | --- | --- | --- | | `fuel` | Fuel price scrapers | Fuel searches, joined to `fuel_prices` | -| `ev_charger` | OpenChargeMap, Mapa REVE and BNetzA scrapers | The `EV` filter, without prices | +| `ev_charger` | OCPDB, OpenChargeMap, Mapa REVE and BNetzA scrapers | The `EV` filter, with the newest `EV` price when one exists | | `both` | Accepted by the scraper contract. No built-in scraper writes it. | The `EV` filter, and fuel searches when it has prices | `geom` stores longitude first, as PostGIS does. To read it back as numbers, use `ST_X(geom)` for longitude and `ST_Y(geom)` for latitude. @@ -127,12 +134,24 @@ One row per price of one fuel at one station. | `id` | `bigint` | No | Row id, from a sequence. | | `station_id` | `uuid` | No | The station, `stations.id`. Deleting a station deletes its prices. | | `fuel_type` | `varchar(20)` | No | A fuel code such as `E5`, `E10` or `B7`. See [Fuel types](fuel-types.md). The database does not check the value. The `RawFuelPrice` type limits scrapers to codes from the list when the code is compiled. | -| `price` | `decimal` | No | Price per litre in `currency`, with three decimals. Gas fuels sold by weight, such as CNG in some countries, are per kilogram. | +| `price` | `decimal` | No | Price per litre in `currency`, with three decimals. Gas fuels sold by weight, such as CNG in some countries, are per kilogram. `EV` prices are per kWh. | | `currency` | `varchar(3)` | No | ISO 4217 code of `price`, such as `EUR` or `SEK`. Defaults to `EUR`. Scrapers always set it. | -| `reported_at` | `timestamptz` | No | When Pumperly wrote the price, not when the source last changed it. | +| `reported_at` | `timestamptz` | No | When Pumperly wrote the price, not when the source last changed it. The OCPDB scraper is the exception: it writes when the tariff's operator feed last refreshed. | | `source` | `varchar(50)` | No | The name of the scraper source that wrote the row. See [`source`](#source). | +| `ev_fees` | `jsonb` | Yes | `EV` rows only: charges on top of the per-kWh price, in `currency`. See [`ev_fees`](#ev_fees). `null` for fuel rows and for tariffs with no extra charges. | -`EV` is a valid fuel code for API requests, but no price row ever carries it. An `EV` search queries stations by `station_type` instead. EV chargers have no rows in this table. +An `EV` search queries stations by `station_type`, then joins each one to its newest `EV` row here, if any. Only the [OCPDB scraper](../data/ev-sources.md#ocpdb) writes `EV` rows, so only German chargers have prices. Every other charger has no rows in this table and shows unpriced. + +### `ev_fees` {#ev_fees} + +The JSON object holds the parts of a charging tariff that a per-kWh price cannot express. Every field is optional. + +| Field | Meaning | +| --- | --- | +| `sessionFee` | A fixed fee per charging session. When the fee varies, this is the highest one, and that is the value cost estimates use. | +| `sessionFeeRules` | Present only when the session fee varies. A list of `{ fee, from?, to?, days?, other? }`: `from` and `to` are local `HH:MM` times, `days` are ISO weekdays (1 is Monday), and `other: true` means further conditions apply. | +| `perMinute` | A blocking fee per minute. | +| `afterMin` | Minutes into the session at which `perMinute` starts. `0` means from the start. | Nothing stops two rows for the same station and fuel. Readers take the newest one by `reported_at`. The example query [below](#reading-the-newest-price) shows the pattern. @@ -168,11 +187,12 @@ To keep sources apart, scrapers prefix the upstream id with a short tag. Some ex | `ocm` (OpenChargeMap, every country) | `ocm-` | | `reve` (Mapa REVE, Spain) | `reve-` | | `bnetza` (BNetzA register, Germany) | `bnetza-_`, rounded to 5 decimals. The register has one row per charging device, and the scraper merges the rows at one position into one station. | +| `ocpdb` (Open ChargePoint Database, Germany) | `ocpdb-_`, rounded to 5 decimals, from the location that anchors the pin. Locations within 25 m of each other merge into one station. | | `miteco` (Spain) | The ministry's station id, with no prefix | Some older scrapers use the upstream id without a prefix, as `miteco` does. A new scraper should use a prefix. [Adding a country](../data/adding-a-country.md#choosing-source-and-externalid) gives the rules. -`stations` has no `source` column. For EV chargers, which have no prices, the `external_id` prefix is the only record of where a row came from. That is how the Mapa REVE and BNetzA scrapers find the OpenChargeMap rows they replace: `external_id LIKE 'ocm-%'`. +`stations` has no `source` column. For EV chargers, most of which have no prices, the `external_id` prefix is the only record of where a row came from. That is how the EV scrapers find the rows of the sources they replace: Mapa REVE deletes `external_id LIKE 'ocm-%'` in Spain, and OCPDB and BNetzA each delete the other German prefixes among `ocm-`, `bnetza-` and `ocpdb-`. ## `source` diff --git a/docs/reference/environment-variables.md b/docs/reference/environment-variables.md index 585abc9..c1c3d9a 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/environment-variables.md @@ -58,7 +58,7 @@ Feature page: [Countries and scrape schedule](../configuration/countries-and-sch Feature page: [Countries and scrape schedule](../configuration/countries-and-schedule.md). -A scraper key names one scheduled scraper. Most are a country code (`ES`). The others are `AU_NSW`, `EV_` for Open Charge Map, `EV_ES_REVE`, `EV_DE_BNETZA` and `STATIC_`. [Scraper keys](../configuration/countries-and-schedule.md#scraper-keys) lists them. +A scraper key names one scheduled scraper. Most are a country code (`ES`). The others are `AU_NSW`, `EV_` for Open Charge Map, `EV_ES_REVE`, `EV_DE_BNETZA`, `EV_DE_OCPDB` and `STATIC_`. [Scraper keys](../configuration/countries-and-schedule.md#scraper-keys) lists them. | Variable | Default | Read by | Notes | |---|---|---|---| @@ -70,7 +70,7 @@ A scraper key names one scheduled scraper. Most are a country code (`ES`). The o Feature page: [How scrapers work](../data/how-scrapers-work.md). -Every scraper run drops prices that cannot be real before it writes anything. A price of zero or less is always dropped. Hydrogen, CNG, LNG and AdBlue prices must be at least 0.05 and below 100, in any currency. +Every scraper run drops prices that cannot be real before it writes anything. A price of zero or less is always dropped. Hydrogen, CNG, LNG and AdBlue prices must be at least 0.05 and below 100, in any currency. EV prices are per kWh and must be in EUR, from 0.10 to 2.00. None of these ranges can be changed. | Variable | Default | Read by | Notes | |---|---|---|---| @@ -106,9 +106,11 @@ Feature pages: [EV charging sources](../data/ev-sources.md) and [API keys](../co | Variable | Default | Read by | Notes | |---|---|---|---| -| `PUMPERLY_EV_ENABLED` | on | `src/instrumentation-node.ts` | Only the exact value `0` turns off every EV charger scraper, including REVE and BNetzA. Any other value leaves them on. It does not depend on whether an Open Charge Map key is set. | -| `PUMPERLY_DE_EV_SOURCE` | `bnetza` | `src/scrapers/germany-ev-source.ts` | Which source supplies German chargers. `ocm`, in any case, picks Open Charge Map. Anything else picks the BNetzA Ladesäulenregister. The two never run together. | -| `PUMPERLY_BNETZA_MIN_STATIONS` | `10000` | `src/scrapers/bnetza.ts` | Stations a BNetzA run must refresh before it deletes chargers that left the register and retires Open Charge Map's German rows. A safety floor: a damaged download cannot empty the map. A value below 1 or not a number means the default. | +| `PUMPERLY_EV_ENABLED` | on | `src/instrumentation-node.ts` | Only the exact value `0` turns off every EV charger scraper, including REVE, BNetzA and OCPDB. Any other value leaves them on. It does not depend on whether an Open Charge Map key is set. | +| `PUMPERLY_DE_EV_SOURCE` | `ocpdb` | `src/scrapers/germany-ev-source.ts` | Which source supplies German chargers, in any case: `ocpdb` for the Open ChargePoint Database, with prices; `bnetza` for the BNetzA Ladesäulenregister; `ocm` for Open Charge Map. Anything else picks OCPDB. Only one ever runs. | +| `PUMPERLY_BNETZA_MIN_STATIONS` | `10000` | `src/scrapers/bnetza.ts` | Stations a BNetzA run must refresh before it deletes chargers that left the register and retires the Open Charge Map and OCPDB German rows. A safety floor: a damaged download cannot empty the map. A value below 1 or not a number means the default. | +| `PUMPERLY_OCPDB_MIN_STATIONS` | `10000` | `src/scrapers/ocpdb.ts` | Stations an OCPDB run must refresh before it deletes chargers that left the database and retires the Open Charge Map and BNetzA German rows. Same safety floor as BNetzA's. A value below 1 or not a number means the default. | +| `PUMPERLY_OCPDB_MAX_FEED_AGE_DAYS` | `3` | `src/scrapers/ocpdb.ts` | Days an OCPDB operator feed may go without refreshing before its tariffs are ignored, so its chargers show unpriced instead of with old prices. Feeds OCPDB marks as failed are always ignored. A value of 0 or less, or not a number, means the default. | | `PUMPERLY_REVE_PAGES_PER_RUN` | `4` | `src/scrapers/reve.ts` | Pages of 100 locations each REVE run fetches. Values above 5 are lowered to 5, the API's hourly limit. A value below 1 or not a number means the default. | | `PUMPERLY_REVE_CUTOVER_RATIO` | `0.95` | `src/scrapers/reve.ts` | Share of the REVE registry that must be stored before Open Charge Map's Spanish rows are deleted. Must be above 0 and at most 1. Anything else means the default. | diff --git a/docs/reference/fuel-types.md b/docs/reference/fuel-types.md index 9f36cef..4890be3 100644 --- a/docs/reference/fuel-types.md +++ b/docs/reference/fuel-types.md @@ -75,16 +75,17 @@ Many sources price CNG, LNG and hydrogen per kilogram. Sweden's feed, for exampl Every price a scraper produces goes through a sanity check before it is stored. Most fuels are checked against a range that depends on the currency. [Currencies and exchange rates](currencies.md#price-bands) has the table. -`CNG`, `LNG`, `H2` and `ADBLUE` skip that table. For these four, any price of at least 0.05 and below 100 is kept, whatever the currency. `LPG` is sold by the litre, so it goes through the normal per-currency range. +`CNG`, `LNG`, `H2` and `ADBLUE` skip that table. For these four, any price of at least 0.05 and below 100 is kept, whatever the currency. `LPG` is sold by the litre, so it goes through the normal per-currency range. `EV` is per kWh and is kept only in EUR, from 0.10 to 2.00. ### EV charging {#ev} -`EV` is not a fuel with a price. It selects stations by type instead: +`EV` selects stations by type, not by price: -- The API returns every station whose type is `ev_charger` or `both`, and ignores `fuel_prices`. -- EV features have no `price` and no `reportedAt`. Their `currency` is `EUR` and carries no meaning. +- The API returns every station whose type is `ev_charger` or `both`, priced or not. +- A charger with an `EV` row in `fuel_prices` carries its newest `price`, per kWh, with `currency`, `reportedAt` and any `evFees`. Only German chargers from OCPDB have these rows. +- Unpriced chargers have no `price` and no `reportedAt`. Their `currency` is `EUR` and carries no meaning. -None of Pumperly's EV sources stores a tariff. Charging tariffs are per kilowatt-hour and per session, and a single per-litre number cannot express them. [EV charging sources](../data/ev-sources.md) lists where the chargers come from, and [EV charging](../using/ev-charging.md) shows how they appear on the map. +A charging tariff is more than a per-kWh price. The session and per-minute fees on top go in `evFees`, described with the [`ev_fees`](data-model.md#ev_fees) column. [EV charging sources](../data/ev-sources.md) lists where the chargers come from, and [EV charging](../using/ev-charging.md) shows how they appear on the map. ## Adding a fuel code diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index dc3c7a4..8e962d9 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -12,7 +12,7 @@ A rectangle on the map, given as four numbers: `minLon,minLat,maxLon,maxLat`. Th ## BNetzA {#bnetza} -The Bundesnetzagentur, Germany's federal network agency. It publishes the Ladesäulenregister, the official register of public charge points in Germany. Every operator of a public charge point must file its chargers there. Pumperly downloads the register as one file, with no API key, and uses it as Germany's default EV source. Its [scraper key](#scraper-key) is `EV_DE_BNETZA`. See [EV charging sources](../data/ev-sources.md). +The Bundesnetzagentur, Germany's federal network agency. It publishes the Ladesäulenregister, the official register of public charge points in Germany. Every operator of a public charge point must file its chargers there. Pumperly can download the register as one file, with no API key, when `PUMPERLY_DE_EV_SOURCE=bnetza`. By default it uses [OCPDB](#ocpdb), which contains the register. Its [scraper key](#scraper-key) is `EV_DE_BNETZA`. See [EV charging sources](../data/ev-sources.md). ## Corridor {#corridor} @@ -45,11 +45,11 @@ The countries an instance scrapes. They come from [`PUMPERLY_ENABLED_COUNTRIES`] ## EV_XX key {#ev-xx-key} -The [scraper key](#scraper-key) for a country's EV chargers from [OpenChargeMap](#ocm), where `XX` is the country code: `EV_FR`, `EV_IT` and so on. Listing `FR` in the [enabled countries](#enabled-countries) turns on `EV_FR` too, unless [`PUMPERLY_EV_ENABLED`](environment-variables.md#pumperly_ev_enabled) is `0`. `EV_US` exists without a matching fuel scraper, so listing `US` gives EV chargers only. Spain and Germany each have a second EV key, `EV_ES_REVE` and `EV_DE_BNETZA`, and only one of each pair ever runs. See [handover](#handover). +The [scraper key](#scraper-key) for a country's EV chargers from [OpenChargeMap](#ocm), where `XX` is the country code: `EV_FR`, `EV_IT` and so on. Listing `FR` in the [enabled countries](#enabled-countries) turns on `EV_FR` too, unless [`PUMPERLY_EV_ENABLED`](environment-variables.md#pumperly_ev_enabled) is `0`. `EV_US` exists without a matching fuel scraper, so listing `US` gives EV chargers only. Spain and Germany have more EV keys: `EV_ES_REVE` for Spain, `EV_DE_OCPDB` and `EV_DE_BNETZA` for Germany. Only one key per country ever runs. See [handover](#handover). ## External ID {#external-id} -A station's identifier at its [source](#source). Together with the country code it is the key a scraper writes on: a re-import updates the same row instead of adding a new one, and share links use this pair. For [BNetzA](#bnetza) chargers the ID is built from the coordinates, so it changes when an operator corrects a position. Some sources get a prefix, such as `ocm-` for [OpenChargeMap](#ocm) and `reve-` for [REVE](#reve). The database's own UUID can change, so do not store it. See [Data model](data-model.md). +A station's identifier at its [source](#source). Together with the country code it is the key a scraper writes on: a re-import updates the same row instead of adding a new one, and share links use this pair. For [BNetzA](#bnetza) and [OCPDB](#ocpdb) chargers the ID is built from the coordinates, so it changes when an operator corrects a position. Some sources get a prefix, such as `ocm-` for [OpenChargeMap](#ocm) and `reve-` for [REVE](#reve). The database's own UUID can change, so do not store it. See [Data model](data-model.md). ## Fuel code {#fuel-code} @@ -65,7 +65,7 @@ A standard JSON format for map data. Pumperly returns stations as a GeoJSON `Fea ## Handover {#handover} -The switch from one [source](#source) to another for the same data, done without leaving the map empty in between. For Spain, [REVE](#reve) takes over EV chargers from [OpenChargeMap](#ocm) once a REVE key is set. REVE fills in slowly, over about a day and a half, so the OpenChargeMap rows stay until REVE holds 95% of its registry by default, and are deleted then. For Germany, [BNetzA](#bnetza) takes over after its first healthy run, because the register arrives complete in one file. That run must refresh a minimum number of stations before it may delete anything, so a broken download cannot remove working rows. Sweden's fuel scraper uses the same kind of floor when it retires the rows of an older source. See [EV charging sources](../data/ev-sources.md). +The switch from one [source](#source) to another for the same data, done without leaving the map empty in between. For Spain, [REVE](#reve) takes over EV chargers from [OpenChargeMap](#ocm) once a REVE key is set. REVE fills in slowly, over about a day and a half, so the OpenChargeMap rows stay until REVE holds 95% of its registry by default, and are deleted then. For Germany, [OCPDB](#ocpdb) or [BNetzA](#bnetza) takes over after its first healthy run, because each arrives complete in one run. That run must refresh a minimum number of stations before it may delete anything, so a broken download cannot remove working rows. Sweden's fuel scraper uses the same kind of floor when it retires the rows of an older source. See [EV charging sources](../data/ev-sources.md). ## Latest price {#latest-price} @@ -79,9 +79,13 @@ Newline-delimited JSON: one complete JSON object per line. [`POST /api/route-det [OpenChargeMap](https://openchargemap.org), a crowdsourced, worldwide registry of EV charge points, published under the Open Database License. It is Pumperly's default EV source for every country except Germany. It needs a free API key in [`PUMPERLY_OCM_API_KEY`](environment-variables.md#pumperly_ocm_api_key). Without one, the OpenChargeMap scrapers fetch nothing. See [EV charging sources](../data/ev-sources.md) and [API keys](../configuration/api-keys.md). +## OCPDB (Open ChargePoint Database) {#ocpdb} + +A German charge point database run by MobiData BW. It merges the [BNetzA](#bnetza) register with the operators' own feeds, which carry ad-hoc prices. It is Germany's default EV source and the only one with prices: the cheapest ad-hoc tariff per kWh, plus any session and per-minute fees. It needs no API key. Its [scraper key](#scraper-key) is `EV_DE_OCPDB`. See [EV charging sources](../data/ev-sources.md#ocpdb). + ## Orphan cleanup {#orphan-cleanup} -The last step of a scraper run. It deletes the country's fuel stations that no longer have a price from any source. EV chargers are never removed this way, because they have no prices. If any batch of station writes failed in the run, the cleanup is skipped. That stops it from deleting stations whose update never landed. +The last step of a scraper run. It deletes the country's fuel stations that no longer have a price from any source. EV chargers are never removed this way, priced or not. If any batch of station writes failed in the run, the cleanup is skipped. That stops it from deleting stations whose update never landed. ## Photon {#photon} @@ -93,7 +97,7 @@ The spatial extension for PostgreSQL. It stores each station's position as a poi ## Price band {#price-band} -The range of prices Pumperly accepts for a currency, such as 200 to 2,000 for Hungarian forints. A scraper drops any price outside its band before storing it. That catches placeholders and prices in the wrong unit. See [Price bands](currencies.md#price-bands). +The range of prices Pumperly accepts for a currency, such as 200 to 2,000 for Hungarian forints. EV prices have their own range: 0.10 to 2.00 EUR per kWh. A scraper drops any price outside its band before storing it. That catches placeholders and prices in the wrong unit. See [Price bands](currencies.md#price-bands). ## Rate limit {#rate-limit} @@ -117,7 +121,7 @@ The code that downloads one [source's](#source) data, turns it into Pumperly's s ## Scraper key {#scraper-key} -The name a scraper is scheduled and configured under. Fuel scrapers use the country code, such as `ES`. EV scrapers use an [EV_XX key](#ev-xx-key). A few have their own names: `AU_NSW` for New South Wales, `EV_ES_REVE`, `EV_DE_BNETZA`, and `STATIC_` for each [static dataset](#static-dataset). The per-scraper interval variable uses the same key, as in `PUMPERLY_SCRAPE_INTERVAL_EV_ES_REVE`. +The name a scraper is scheduled and configured under. Fuel scrapers use the country code, such as `ES`. EV scrapers use an [EV_XX key](#ev-xx-key). A few have their own names: `AU_NSW` for New South Wales, `EV_ES_REVE`, `EV_DE_BNETZA`, `EV_DE_OCPDB`, and `STATIC_` for each [static dataset](#static-dataset). The per-scraper interval variable uses the same key, as in `PUMPERLY_SCRAPE_INTERVAL_EV_ES_REVE`. ## Source {#source} diff --git a/docs/roadmap.md b/docs/roadmap.md index 9d8c669..418056d 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -30,8 +30,8 @@ What works today is described in [The map](using/map.md), [Planning a route](usi ## EV charging - **EV route planning.** You enter battery size, charge level and consumption, and Pumperly plans the charging stops. Today EV chargers show on the map and along a route, like fuel stations, without range planning. -- **Connector, power and price details.** Each charger shows its connector types, power in kW and price per kWh. Today a charger is stored like any station, with its location, name, brand and address, and none of these fields. -- **More official registries.** National registries replace crowdsourced data in more countries. Today Spain (Mapa REVE, with an API key) and Germany (the BNetzA Ladesäulenregister) use official registries, and every other country uses Open Charge Map. See [EV charging sources](data/ev-sources.md). +- **Connector and price details everywhere.** Each charger shows its connector types, power in kW and price per kWh. Today every charger shows its power, but only German chargers have prices, from OCPDB, and none has connector types. +- **More official registries.** National registries replace crowdsourced data in more countries. Today Spain (Mapa REVE, with an API key) and Germany (OCPDB, which contains the BNetzA Ladesäulenregister) use official sources, and every other country uses Open Charge Map. See [EV charging sources](data/ev-sources.md). ## Coverage diff --git a/docs/using/ev-charging.md b/docs/using/ev-charging.md index 88e07b2..bd5a538 100644 --- a/docs/using/ev-charging.md +++ b/docs/using/ev-charging.md @@ -16,13 +16,13 @@ Pumperly keeps a type on every station: a fuel station, an EV charger, or both. The fuel layers work differently. They show a station only when it has a price for the chosen fuel. A station marked as both appears on the EV layer, and on each fuel layer it has a price for. -Chargers carry no price. That changes a few things on the map: +Only German chargers have prices, per kWh, from [OCPDB](../data/ev-sources.md#ocpdb). Chargers elsewhere carry none. That changes a few things on the map: | On a fuel layer | On the EV layer | |-----------------|-----------------| -| Dots are coloured by price | Every dot is grey, and every cluster is orange | -| The price legend shows in the corner | No legend | -| The max-price slider shows | No max-price slider | +| Dots are coloured by price | Priced chargers are coloured by price per kWh. Unpriced chargers are grey. With no priced charger in view, every dot is grey and every cluster is orange | +| The price legend shows in the corner | The legend shows when chargers in view have prices | +| The max-price slider shows | The slider shows when at least two chargers in view have different prices. It hides priced chargers above the limit and keeps unpriced ones | Clustering, country markers and the Stats panel work as on any other layer. The station counts in the country markers and the Stats panel include chargers. @@ -33,7 +33,7 @@ Pumperly uses an official national registry where one exists, and [Open Charge M | Where | Source | |-------|--------| | Spain | Mapa REVE, the official registry, when `PUMPERLY_REVE_API_KEY` is set. Open Charge Map otherwise. | -| Germany | The BNetzA Ladesäulenregister by default. `PUMPERLY_DE_EV_SOURCE=ocm` switches to Open Charge Map. | +| Germany | The Open ChargePoint Database (OCPDB) by default, with prices. `PUMPERLY_DE_EV_SOURCE=bnetza` switches to the BNetzA Ladesäulenregister, and `ocm` to Open Charge Map. Neither has prices. | | Every other covered country | Open Charge Map | Open Charge Map needs `PUMPERLY_OCM_API_KEY`. Without it, every Open Charge Map scraper skips its run. That includes Spain without a REVE key, and Germany with `PUMPERLY_DE_EV_SOURCE=ocm`. @@ -46,19 +46,20 @@ Click a charger to open its popup. It holds: 1. The **brand**. For chargers this is the operator, when the source names one. 2. The **address** and the **city**. -3. The charger's **highest power** in kW, such as `150 kW`, labelled "Max. charging power · No price for EV Charging". When the source lists no power, it reads "Power unknown" instead. The German registry, OpenChargeMap and Mapa REVE all publish power. -4. The same four buttons as any station: **Get directions**, **Show on map**, **Copy link** and **Share**. See [Station popup](map.md#station-popup). +3. For a priced charger, the **price per kWh**, with the charger's highest power and how old the price is. Fees on top follow on one line, such as `+0.20 €/min after 45 min · +0.50 € per session`. When the session fee depends on the time or day, each option is listed with when it applies, such as `+0.50 € per session (06:00–22:00) · +1.00 € per session (22:00–06:00)`. +4. For an unpriced charger, its **highest power** in kW, such as `150 kW`, labelled "Max. charging power · Price not published". When the source lists no power, it reads "Power unknown" instead. Every charger source publishes power. +5. The same four buttons as any station: **Get directions**, **Show on map**, **Copy link** and **Share**. See [Station popup](map.md#station-popup). In a route's station list, a charger row shows the operator and the station name. Each source builds that name differently. The German registry, for example, combines the operator with the street and a site label, which tells apart two chargers at one address. -!!! note "No tariffs, connectors or live availability" - Pumperly does not store charging tariffs, connector types or live availability. Its price model holds one price per litre for each fuel. A charging tariff is per kWh and per session, and that does not fit. The popup tells you where a charger is, who runs it and how fast it can charge. Check the operator's app for price and plug type. +!!! note "Ad-hoc prices only, no connectors or live availability" + The price shown is the cheapest ad-hoc tariff, what a driver without a contract pays. Contract and subscription prices are not stored, and neither are connector types or live availability. Outside Germany no source provides prices. Check the operator's app for those. ## Chargers along a route Route planning on the EV layer works the same way as on a fuel layer. See [Planning a route](routes.md). Pumperly finds the chargers within the search radius of each route and times the detour to each one. -The difference is that no charger has a price. That affects the station list: +The difference is that many chargers have no price. That affects the station list: | Feature | On the EV layer | |---------|-----------------| @@ -66,19 +67,23 @@ The difference is that no charger has a price. That affects the station list: | Detour times and the detour basis | Work as usual | | Max detour | Works as usual | | Sort by **Detour** or **Route km** | Works as usual | -| Sort by **Price** | Keeps the chargers in route order, since none has a price | +| Sort by **Price** | Priced chargers first, cheapest per kWh first. Unpriced chargers follow in route order | | **Least detour** badge | Works as usual | -| **Cheapest** and **Balanced** badges | Never shown | -| Average price and the difference from it | Not shown | +| **Cheapest** and **Balanced** badges | Among priced chargers only. Never shown when none has a price | +| Average price and the difference from it | Per kWh, over the priced chargers. Not shown when none has a price | -Every charger within the radius is included, because none is left out for lacking a price. A route gets at most 5,000 stations, in route order. +Every charger within the radius is included, priced or not. Unlike fuel stations, none is left out for lacking a price. A route gets at most 5,000 stations, in route order. !!! tip "Keep the radius small in cities" Cities have far more chargers than fuel stations. A 5 km radius across a large city can hold hundreds of them. Pumperly times detours in batches of 150 chargers, and each batch is one request. A visitor may send 10 detour requests a minute, so the later batches of a very dense corridor come back as unknown detours. Lower the search radius, or pick a route that avoids the centre. The limits are listed in [Limits](routes.md#limits). Clicking a charger in the list or on the map adds it to the route as a stop, as on any layer. See [Pick a station as a stop](routes.md#pick-a-station-as-a-stop). -On the EV layer, the fuel stop planner becomes **Plan charging stops**. You set the battery size in kWh, the consumption in kWh/100 km, the car's max charge power, and the battery level at the start and at arrival. The planner then picks the chargers along the route that get you there with the least detour and charging time, since chargers have no prices. Each stop charges to at most 80 %, because charging slows sharply above that. Charging time uses the lower of the charger's power and the car's max charge power, capped at 22 kW on AC chargers. On DC it averages 75 % of that, because charging tapers as the battery fills. The reserve defaults to 20 %. **Min charger power** (Any, 50 kW or 150 kW) skips slower chargers. It also skips chargers whose power is unknown, because the filter can't vouch for them. +On the EV layer, the fuel stop planner becomes **Plan charging stops**. You set the battery size in kWh, the consumption in kWh/100 km, the car's max charge power, and the battery level at the start and at arrival. + +When any charger along the route has a price, the planner uses only priced chargers and picks the stops with the lowest cost: the energy at each charger's price per kWh, plus its session fee, plus your time, priced with **Value of your time**. Each stop shows its cost, and so does the plan. A session fee that varies counts at its highest. Per-minute blocking fees are left out. When no charger along the route has a price, as outside Germany, the planner picks the stops with the least detour and charging time. + +Each stop charges to at most 80 %, because charging slows sharply above that. Charging time uses the lower of the charger's power and the car's max charge power, capped at 22 kW on AC chargers. On DC it averages 75 % of that, because charging tapers as the battery fills. The reserve defaults to 20 %. **Min charger power** (Any, 50 kW or 150 kW) skips slower chargers. It also skips chargers whose power is unknown, because the filter can't vouch for them. ## Sharing a charger @@ -87,7 +92,8 @@ A charger link must carry `fuel=EV`. The map only loads chargers on the EV layer ## Summary - Pick **EV Charging** (`EV`) in the fuel selector to see chargers. -- The EV layer shows every charger in view. Chargers have no price, so dots are grey and there is no legend or price slider. -- A charger's popup shows the operator, the address, the city and the highest power in kW. There are no tariffs, connectors or live availability. -- Along a route, detours and the detour filter work as usual. Price sorting and the Cheapest and Balanced badges do not apply. +- The EV layer shows every charger in view. German chargers have ad-hoc prices per kWh and are coloured by price. Chargers elsewhere are grey, with no legend or price slider. +- A charger's popup shows the operator, the address, the city, the highest power in kW, and for priced chargers the price per kWh and any session or per-minute fees. There are no connectors or live availability. +- Along a route, detours and the detour filter work as usual. Price sorting and the Cheapest and Balanced badges use the priced chargers. +- The charging stop planner costs stops by price and session fee when the route has priced chargers, and by time alone when it has none. - Keep the search radius small in cities, or dense corridors run into the detour rate limit. diff --git a/docs/using/links.md b/docs/using/links.md index eba1c2c..7f20b5c 100644 --- a/docs/using/links.md +++ b/docs/using/links.md @@ -87,7 +87,7 @@ A station link without a valid `lat` and `lng` is ignored, and the map opens as **A link to an EV charger must carry `fuel=EV`.** -The map loads stations for one fuel at a time. On a fuel layer it loads only stations with a price for that fuel. Chargers have no price, so they load only on the EV layer. +The map loads stations for one fuel at a time. On a fuel layer it loads only stations with a price for that fuel. Chargers have no fuel price, so they load only on the EV layer. A charger link without `fuel=EV` opens on the instance's default fuel. The charger never loads, so the popup never opens. You land on the right spot, but the charger is not there. diff --git a/docs/using/map.md b/docs/using/map.md index 01cdd56..c408b2e 100644 --- a/docs/using/map.md +++ b/docs/using/map.md @@ -70,7 +70,7 @@ Each fuel has a code, such as `B7` for regular diesel or `EV` for charging. The The first fuel you see is the instance default. The operator sets it with [`PUMPERLY_DEFAULT_FUEL`](../reference/environment-variables.md#pumperly_default_fuel). Without it, Pumperly uses the default fuel of the country in [`PUMPERLY_DEFAULT_COUNTRY`](../reference/environment-variables.md#pumperly_default_country). -On a fuel layer, a station appears only when it has a price for that fuel. A station that sells diesel but reported no diesel price does not show on the diesel layer. The EV layer works differently: it shows every charger, and chargers carry no price. See [EV charging](ev-charging.md). +On a fuel layer, a station appears only when it has a price for that fuel. A station that sells diesel but reported no diesel price does not show on the diesel layer. The EV layer works differently: it shows every charger, priced or not. Only German chargers have prices. See [EV charging](ev-charging.md). Changing the fuel clears the max-price filter and resets the max detour to 5 minutes. From e56675a3f8c0a6ddf6adbe7aaf82223ccd51b873 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:22:52 +0200 Subject: [PATCH 09/25] docs: stop calling every charger unpriced --- docs/reference/fuel-types.md | 2 +- docs/using/links.md | 2 +- docs/using/map.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/fuel-types.md b/docs/reference/fuel-types.md index 4890be3..a10403d 100644 --- a/docs/reference/fuel-types.md +++ b/docs/reference/fuel-types.md @@ -26,7 +26,7 @@ The core codes, such as `E5`, `E10`, `B7`, `B10`, `LPG`, `CNG`, `LNG` and `H2`, | `CNG` | GNC | gas | Compressed natural gas, including biomethane sold as CNG. | | `LNG` | GNL | gas | Liquefied natural gas, mostly for trucks. | | `H2` | Hidrógeno | hydrogen | Hydrogen for fuel-cell vehicles. | -| `EV` | EV Charging | electric | Electric vehicle charge points. Has no price. See [below](#ev). | +| `EV` | EV Charging | electric | Electric vehicle charge points. Priced per kWh where the source publishes it. See [below](#ev). | | `ADBLUE` | AdBlue | other | Diesel exhaust fluid, the urea solution diesel engines inject into their exhaust. | The labels are fixed strings. They show as written in every language the app supports. Only the category headings in the fuel picker are translated. diff --git a/docs/using/links.md b/docs/using/links.md index 7f20b5c..d4e6ce4 100644 --- a/docs/using/links.md +++ b/docs/using/links.md @@ -87,7 +87,7 @@ A station link without a valid `lat` and `lng` is ignored, and the map opens as **A link to an EV charger must carry `fuel=EV`.** -The map loads stations for one fuel at a time. On a fuel layer it loads only stations with a price for that fuel. Chargers have no fuel price, so they load only on the EV layer. +The map loads stations for one fuel at a time. A layer other than EV loads only stations with a price for that fuel. A charger's only price, when it has one, is an `EV` price, so it loads only on the EV layer. A charger link without `fuel=EV` opens on the instance's default fuel. The charger never loads, so the popup never opens. You land on the right spot, but the charger is not there. diff --git a/docs/using/map.md b/docs/using/map.md index c408b2e..c6bdcee 100644 --- a/docs/using/map.md +++ b/docs/using/map.md @@ -154,7 +154,7 @@ From top to bottom the popup shows: | Copy link | Copies a link to this station to the clipboard. The icon turns into a check mark when it worked. | | Share | Opens your device's share sheet. Where the browser has none, it copies the link instead. | -When the station has no price for the chosen fuel, the price block reads "No price for" and the fuel name. This is what an EV charger shows. +When the station has no price for the chosen fuel, the price block reads "No price for" and the fuel name. An EV charger shows its price per kWh when it has one, and its power otherwise. See [What a charger shows](ev-charging.md#what-a-charger-shows). The two link buttons are described in [Share links and deep links](links.md). From f7262d8f931a7af1a85490d3955be3af1c1317b2 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:25:07 +0200 Subject: [PATCH 10/25] fix(planner): weigh stop fees when pruning candidates --- src/lib/refuel-planner.test.ts | 9 +++++++++ src/lib/refuel-planner.ts | 12 ++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/lib/refuel-planner.test.ts b/src/lib/refuel-planner.test.ts index 4f2f472..230497f 100644 --- a/src/lib/refuel-planner.test.ts +++ b/src/lib/refuel-planner.test.ts @@ -313,6 +313,15 @@ describe("pruneCandidates", () => { expect(pruneCandidates(stations, 1000, 7).length).toBeLessThanOrEqual(7); }); + it("weighs the stop fee when picking a bucket's cheapest", () => { + // Over a 40 kWh charge "fee" costs 0.39 + 10/40 = 0.64 per kWh, "nofee" 0.45. + const fee = { ...st("fee", 0.1, 0.39, 5), stopFee: 10 }; + const nofee = st("nofee", 0.2, 0.45, 5); + const fillers = Array.from({ length: 250 }, (_, i) => st(`F${i}`, i * 0.001, 0.6, 1)); + expect(pruneCandidates([...fillers, fee, nofee], 1000, 200, 40).map((s) => s.id)).toContain("nofee"); + expect(pruneCandidates([...fillers, fee, nofee], 1000, 200).map((s) => s.id)).not.toContain("nofee"); + }); + it("returns the input untouched under the cap", () => { const stations = [st("A", 1, 1), st("B", 2, 1)]; expect(pruneCandidates(stations, 10, 200)).toBe(stations); diff --git a/src/lib/refuel-planner.ts b/src/lib/refuel-planner.ts index f09d44a..8c59ee7 100644 --- a/src/lib/refuel-planner.ts +++ b/src/lib/refuel-planner.ts @@ -100,8 +100,11 @@ const L = 101; // fuel levels 0..100 % * only station a low tank can reach; the furthest-reaching one means pruning never * drops the station needed to get out of the bucket towards the next one. At most * four per bucket keeps the total within `max`. Order by km is preserved. + * "Cheapest" spreads each stop fee over `refQty` litres (kWh), a typical stop's + * purchase, so a low price with a large session fee doesn't crowd out a dearer + * charger without one. With `refQty` 0 only the price counts. */ -export function pruneCandidates(stations: PlannerStation[], routeKm: number, max = MAX_CANDIDATES): PlannerStation[] { +export function pruneCandidates(stations: PlannerStation[], routeKm: number, max = MAX_CANDIDATES, refQty = 0): PlannerStation[] { if (stations.length <= max) return stations; const buckets = Math.max(1, Math.floor(max / 4)); const size = routeKm / buckets || 1; @@ -119,10 +122,12 @@ export function pruneCandidates(stations: PlannerStation[], routeKm: number, max const reachKm = (s: PlannerStation) => s.km + halfDetourKm(s); // Where a full tank bought here effectively starts from, on the route. const onwardKm = (s: PlannerStation) => s.km - halfDetourKm(s); + // Price per litre (kWh) with the stop fee spread over a typical purchase. + const unitCost = (s: PlannerStation) => s.price + (refQty > 0 ? (s.stopFee ?? 0) / refQty : 0); for (const list of byBucket.values()) { // Ties go to the faster charger: unpriced EV plans set every price to 0, so there "cheapest" means "fastest". const cheapest = list.reduce((a, b) => - b.price < a.price || (b.price === a.price && (b.minutesPerPct ?? 0) < (a.minutesPerPct ?? 0)) ? b : a); + unitCost(b) < unitCost(a) || (unitCost(b) === unitCost(a) && (b.minutesPerPct ?? 0) < (a.minutesPerPct ?? 0)) ? b : a); const closest = list.reduce((a, b) => (b.detourMin < a.detourMin ? b : a)); const earliest = list.reduce((a, b) => (reachKm(b) < reachKm(a) ? b : a)); const furthest = list.reduce((a, b) => (onwardKm(b) > onwardKm(a) ? b : a)); @@ -187,6 +192,9 @@ export function planRefuel(input: PlannerInput): PlanResult { .filter((s) => s.km >= 0 && s.km <= routeKm && s.price >= 0 && s.detourMin >= 0) .sort((a, b) => a.km - b.km), routeKm, + MAX_CANDIDATES, + // A stop's purchase is taken as a fill from the reserve to the charge cap. + Math.max(0, topG - reservePct) * litresPerPct, ); const n = stations.length; const detourKm = stations.map((s) => (s.detourMin / 60) * DETOUR_KMH); From 62a6cf14a9b442225eff897169f795151857c764 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:55:48 +0200 Subject: [PATCH 11/25] fix(ocpdb): read VAT given as a fraction or as zero --- src/scrapers/ocpdb.test.ts | 41 ++++++++++++++++++++++++++++++++++++-- src/scrapers/ocpdb.ts | 15 +++++++------- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/scrapers/ocpdb.test.ts b/src/scrapers/ocpdb.test.ts index 6cbe4f5..1fd3fd9 100644 --- a/src/scrapers/ocpdb.test.ts +++ b/src/scrapers/ocpdb.test.ts @@ -86,6 +86,15 @@ describe("grossPrice", () => { expect(grossPrice({ type: "ENERGY", price: 0.42 }, { tax_included: "NO" })).toBe(0.4998); expect(grossPrice({ type: "ENERGY", price: 0.42, taxes: [], tax_included: false }, { tax_included: null })).toBe(0.4998); }); + + // Values as the schema hands them over; the raw payloads are in the fetch tests. + it("reads a rate given as a fraction as a percentage", () => { + expect(grossPrice({ type: "ENERGY", price: 0.66386555, taxes: [{ percentage: 0.19 }], tax_included: false }, { tax_included: "NO" })).toBe(0.79); + }); + + it("adds German VAT to a net price whose rate is 0", () => { + expect(grossPrice({ type: "ENERGY", price: 0.5798, taxes: [{ percentage: 0 }], tax_included: false }, { tax_included: "NO" })).toBe(0.69); + }); }); describe("summarizeTariff", () => { @@ -403,7 +412,9 @@ describe("OcpdbScraper.fetch", () => { const page = (items: unknown[], total = items.length) => json({ items, total_count: total }); /** Routes each URL to a page; records every URL requested. */ - function stubApi(opts: { sources?: unknown[]; locationsTotal?: number; failOnce?: string; failAlways?: string } = {}) { + function stubApi( + opts: { sources?: unknown[]; tariffs?: unknown[]; locationsTotal?: number; failOnce?: string; failAlways?: string } = {}, + ) { const urls: string[] = []; const failed = new Set(); vi.stubGlobal( @@ -426,7 +437,7 @@ describe("OcpdbScraper.fetch", () => { ], }); } - if (url.includes("/tariffs?")) return page([ENBW, CHARGECLOUD]); + if (url.includes("/tariffs?")) return page(opts.tariffs ?? [ENBW, CHARGECLOUD]); if (url.includes("/tariff-associations?")) { return page([ { tariff_id: ENBW.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "enbw" }] }, @@ -468,6 +479,32 @@ describe("OcpdbScraper.fetch", () => { expect(vi.mocked(console.log).mock.calls.flat().join(" ")).toMatch(/1 unparseable/); }); + // TotalEnergies tariffs on EcoMovement (132622, 132721, 132724, 133685) give + // the rate as a fraction; 132725 gives a net price with a rate of "0". Both + // are 0.79 / 0.69 gross. The ids stand in for the stub's associations. + it("prices the live VAT shapes gross", async () => { + const net = (price: number, percentage: string) => ({ + type: "ENERGY", + price, + taxes: [{ percentage }], + tax_included: false, + step_size: 1, + }); + stubApi({ + locationsTotal: 1001, + tariffs: [ + { id: ENBW.id, source: "datex2_enbw", currency: "EUR", tax_included: "NO", elements: [{ price_components: [net(0.66386555, "0.19000000000000000000")] }] }, + { id: CHARGECLOUD.id, source: "datex2_chargecloud", currency: "EUR", tax_included: "NO", elements: [{ price_components: [net(0.5798, "0")] }] }, + ], + }); + const { OcpdbScraper } = await import("./ocpdb"); + vi.spyOn(console, "log").mockImplementation(() => {}); + + const { prices } = await new OcpdbScraper().fetch(); + + expect(prices.map((p) => p.price).sort()).toEqual([0.69, 0.79]); + }); + it("leaves a dead feed's chargers on the map without a price", async () => { stubApi({ locationsTotal: 2, diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts index b778787..d0e7494 100644 --- a/src/scrapers/ocpdb.ts +++ b/src/scrapers/ocpdb.ts @@ -38,8 +38,9 @@ import { otherGermanyEvPrefixes } from "./germany-ev-source"; // prices with `taxes: [VAT 19%]` and `tax_included: false`, Wirelane // publishes gross, Chargecloud publishes no tax data at all. Ad-hoc // charging is a consumer price and PAngV requires it gross, so "no tax -// data" is read as gross. A price marked net with no rate gets German -// VAT. See grossPrice(). +// data" is read as gross. A price marked net with no rate, or a rate of +// 0, gets German VAT. Some EcoMovement tariffs (TotalEnergies) give the +// rate as a fraction ("0.19"), which is read as 19 %. See grossPrice(). // // 3. TIME components read as per minute here, not OCPI's per hour. Every // one seen is a blocking fee starting 30-240 min in (EnBW 0.2017 net @@ -204,12 +205,10 @@ export interface TariffSummary { /** A component's price with VAT, rounded to 4 dp. See header note 2. */ export function grossPrice(component: PriceComponent, tariff: Pick): number { const included = component.tax_included ?? (tariff.tax_included === "NO" ? false : true); - // Marked net but no rate given: the price is still net, so add German VAT. - const vat = included - ? 0 - : component.taxes?.length - ? component.taxes.reduce((sum, t) => sum + t.percentage, 0) - : DE_VAT_PCT; + // A rate below 1 is a fraction ("0.19"), not a fraction of a percent. + const rate = (component.taxes ?? []).reduce((sum, t) => sum + (t.percentage < 1 ? t.percentage * 100 : t.percentage), 0); + // Marked net but no rate (or 0) given: the price is still net, so add German VAT. + const vat = included ? 0 : rate > 0 ? rate : DE_VAT_PCT; return Math.round(component.price * (1 + vat / 100) * 10_000) / 10_000; } From 9af1c5246eba0573d96801abef6f72b15b7134e5 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:57:04 +0200 Subject: [PATCH 12/25] fix(ocpdb): price a pin from the EVSEs at its top power --- src/scrapers/ocpdb.test.ts | 52 +++++++++++++++++++++++--- src/scrapers/ocpdb.ts | 76 ++++++++++++++++++++++++++------------ 2 files changed, 99 insertions(+), 29 deletions(-) diff --git a/src/scrapers/ocpdb.test.ts b/src/scrapers/ocpdb.test.ts index 1fd3fd9..2144c34 100644 --- a/src/scrapers/ocpdb.test.ts +++ b/src/scrapers/ocpdb.test.ts @@ -323,14 +323,56 @@ describe("mergeLocations", () => { expect(prices[0]).toMatchObject({ stationExternalId: stations[0].externalId, price: 0.46 }); }); - it("keeps the cheapest price across merged locations, and the highest power", () => { - const fast = location({ source: "datex2_enbw", evses: ["enbw"] }); - fast.charging_pool[0].evses[0].connectors = [{ max_electric_power: 300000 }]; + /** A location whose EVSEs (uid → kW) are given explicitly. */ + function site(evses: Record, overrides: Parameters[0] = {}): Location { + const loc = location({ evses: Object.keys(evses), ...overrides }); + loc.charging_pool[0].evses.forEach((e) => (e.connectors = [{ max_electric_power: evses[e.uid] * 1000 }])); + return loc; + } + + // Like ocpdb-51.14720_7.32149 (EnBW Hückeswagen): DC at 0.79, AC at 0.70. + // The pin shows 150 kW, so it must show the 150 kW price. + const hpc = evseTariffs( + [ENBW, { ...ENBW, id: "ac", elements: [{ price_components: [{ type: "ENERGY", price: 0.7, tax_included: true }] }] }], + [ + { tariff_id: ENBW.id, audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "dc" }] }, + { tariff_id: "ac", audience: "AD_HOC_PAYMENT", evses: [{ evse_uid: "ac" }] }, + ], + ); + + it("prices a pin from the EVSEs at its top power, not the cheapest slower one", () => { + const { stations, prices } = mergeLocations([site({ dc: 150, ac: 22 })], hpc); + expect(stations[0].maxPowerKw).toBe(150); + expect(prices[0]).toMatchObject({ price: 0.79, evFees: { perMinute: 0.24, afterMin: 30 } }); + }); + + it("falls back to the cheapest EVSE when none at the top power is priced", () => { + const { stations, prices } = mergeLocations([site({ unpriced: 150, ac: 22 })], hpc); + expect(stations[0].maxPowerKw).toBe(150); + expect(prices[0].price).toBe(0.7); + }); + + it("applies the same rule across merged locations", () => { const { stations, prices } = mergeLocations( - [fast, location({ source: "datex2_chargecloud", lon: 9.1829 + 5 * METRES, evses: ["cc"] })], + [ + site({ ac: 22 }), + site({ dc: 150 }, { source: "datex2_enbw", lon: 9.1829 + 5 * METRES, operator: { name: "EnBW AG" } }), + ], + hpc, + ); + expect(stations[0].maxPowerKw).toBe(150); + expect(prices[0].price).toBe(0.79); + expect(stations[0].brand).toBe("EnBW AG"); + }); + + it("keeps the cheapest price across merged locations of the same power", () => { + const { prices } = mergeLocations( + [ + location({ source: "datex2_enbw", evses: ["enbw"] }), + location({ source: "datex2_chargecloud", lon: 9.1829 + 5 * METRES, evses: ["cc"] }), + ], tariffs, ); - expect(stations[0].maxPowerKw).toBe(300); expect(prices).toHaveLength(1); expect(prices[0].price).toBe(0.46); }); diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts index d0e7494..429ac0f 100644 --- a/src/scrapers/ocpdb.ts +++ b/src/scrapers/ocpdb.ts @@ -363,24 +363,41 @@ export interface OcpdbParseStats { priced: number; } +/** A tariff and the operator of the location it came from. */ +interface Offer { + tariff: TariffSummary; + operator: string | null; +} + interface Pin { station: RawStation; - tariff: TariffSummary | null; + /** Cheapest tariff on the EVSEs at the pin's top power. */ + top: Offer | null; + /** Cheapest tariff on any EVSE: the fallback when no top-power EVSE is priced. */ + any: Offer | null; /** The location's own name or street, which the pin's name is built from. */ site: string; } +function cheaperOffer(a: Offer | null, b: Offer | null): Offer | null { + if (!a) return b; + return cheaper(a.tariff, b?.tariff) ? a : b; +} + /** "Operator — site", falling back to whichever part exists, then the key. */ function pinName(operator: string | null, site: string, key: string): string { return [operator, site].filter(Boolean).join(" — ") || key; } /** - * Turn OCPDB locations into deduplicated pins, each with its cheapest ad-hoc - * tariff. A pin is labelled with the operator whose tariff it shows: when a - * merged location supplies the price, its operator takes over the brand, so a - * driver never sees one operator's name next to another's price. Pure: the merge, the filters and the derived externalId all live - * here, so this is what the tests cover. + * Turn OCPDB locations into deduplicated pins, each with one ad-hoc tariff: + * the cheapest on the EVSEs at the pin's top power, so the price shown is the + * one paid at the kW shown next to it (a 22 kW rate must not pass for the + * 150 kW one). Only when none of those EVSEs is priced does the cheapest + * tariff on any EVSE stand in. A pin is labelled with the operator whose + * tariff it shows, so a driver never sees one operator's name next to + * another's price. Pure: the merge, the filters and the derived externalId + * all live here, so this is what the tests cover. */ export function mergeLocations( locations: OcpdbLocation[], @@ -420,16 +437,19 @@ export function mergeLocations( continue; } + const operator = loc.operator?.name?.trim() || null; // max_electric_power is in W. - const powerKw = maxSanePowerKw( - evses.flatMap((e) => e.connectors.map((c) => (c.max_electric_power ?? 0) / 1000)), - ); - let tariff: TariffSummary | null = null; - for (const e of evses) { - const t = tariffsByEvse.get(e.uid); - if (t && cheaper(t, tariff)) tariff = t; + const evsePower = evses.map((e) => maxSanePowerKw(e.connectors.map((c) => (c.max_electric_power ?? 0) / 1000))); + const powerKw = maxSanePowerKw(evsePower); + let top: Offer | null = null; + let any: Offer | null = null; + for (let i = 0; i < evses.length; i++) { + const t = tariffsByEvse.get(evses[i].uid); + if (!t) continue; + const offer = { tariff: t, operator }; + any = cheaperOffer(offer, any); + if (powerKw != null && evsePower[i] === powerKw) top = cheaperOffer(offer, top); } - const operator = loc.operator?.name?.trim() || null; // Fold into an existing pin within the merge radius, if any. const gy = Math.floor(latitude / GRID_DEG); @@ -450,14 +470,15 @@ export function mergeLocations( if (target) { stats.merged++; - if (powerKw != null && powerKw > (target.station.maxPowerKw ?? 0)) target.station.maxPowerKw = powerKw; - if (tariff && cheaper(tariff, target.tariff)) { - target.tariff = tariff; - if (operator) { - target.station.brand = operator; - target.station.name = pinName(operator, target.site, target.station.externalId); - } + // Same rule as within a location: a faster location brings its own + // top-power price; an equally fast one competes on price. + if (powerKw != null && powerKw > (target.station.maxPowerKw ?? 0)) { + target.station.maxPowerKw = powerKw; + target.top = top; + } else if (powerKw != null && powerKw === target.station.maxPowerKw) { + target.top = cheaperOffer(top, target.top); } + target.any = cheaperOffer(any, target.any); continue; } @@ -479,7 +500,8 @@ export function mergeLocations( stationType: "ev_charger", maxPowerKw: powerKw, }, - tariff, + top, + any, site, }; pins.push(pin); @@ -490,8 +512,14 @@ export function mergeLocations( } const prices: RawFuelPrice[] = []; - for (const { station, tariff } of pins) { - if (!tariff) continue; + for (const { station, top, any, site } of pins) { + const offer = top ?? any; + if (!offer) continue; + const { tariff, operator } = offer; + if (operator && operator !== station.brand) { + station.brand = operator; + station.name = pinName(operator, site, station.externalId); + } stats.priced++; prices.push({ stationExternalId: station.externalId, From e13889f7169698cc8664261f4617b3f76b8579f8 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:58:05 +0200 Subject: [PATCH 13/25] fix(planner): keep unpriced chargers in priced EV plans --- src/components/search/refuel-planner.test.tsx | 33 +++++++++++++++++-- src/components/search/refuel-planner.tsx | 15 +++++---- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/components/search/refuel-planner.test.tsx b/src/components/search/refuel-planner.test.tsx index c2047de..03fb60e 100644 --- a/src/components/search/refuel-planner.test.tsx +++ b/src/components/search/refuel-planner.test.tsx @@ -419,12 +419,11 @@ describe("RefuelPlanner", () => { const chargers = [ makeStation("x", { brand: "Ionity", price: 0.69, fuelType: "EV", routeFraction: 0.3, detourMin: 0, evFees: { sessionFee: 1.5 } }), makeStation("y", { brand: "Tesla", price: 0.49, fuelType: "EV", routeFraction: 0.6, detourMin: 0 }), - makeStation("z", { brand: "Unpriced", price: null, currency: undefined, fuelType: "EV", routeFraction: 0.45, detourMin: 0 }), ]; renderPlanner({ mode: "ev", stations: chargers, routeKm: 300 }); await userEvent.click(screen.getByText("planner.titleEv")); const input = planSpy.mock.lastCall![0]; - // Priced chargers only, at their price, with the real value of time. + // At their price, with the real value of time. expect(input.stations.map((s: { id: string }) => s.id)).toEqual(["x", "y"]); expect(input.stations[0]).toMatchObject({ price: 0.69, stopFee: 1.5 }); expect(input.stations[1].stopFee).toBeUndefined(); @@ -433,6 +432,36 @@ describe("RefuelPlanner", () => { expect(screen.getAllByText(/^\d+\.\d\d € · \+\d+\.\d kWh$/)).toHaveLength(2); }); + // Munich to Vienna: German chargers carry OCPDB prices, Austrian ones none. + it("keeps unpriced chargers in a priced plan, at 0 and with no cost shown", async () => { + const chargers = [ + makeStation("de", { brand: "EnBW", price: 0.79, fuelType: "EV", routeFraction: 0.3, detourMin: 0 }), + makeStation("at", { brand: "Smatrics", price: null, currency: undefined, fuelType: "EV", routeFraction: 0.6, detourMin: 0 }), + ]; + const { onPlanChange } = renderPlanner({ mode: "ev", stations: chargers, routeKm: 300 }); + await userEvent.click(screen.getByText("planner.titleEv")); + const input = planSpy.mock.lastCall![0]; + expect(input.stations).toEqual([ + expect.objectContaining({ id: "de", price: 0.79 }), + expect.objectContaining({ id: "at", price: 0 }), + ]); + expect(input.timeValuePerHour).toBe(15); + expect(onPlanChange.mock.lastCall![0].map((s: { id: string }) => s.id)).toEqual(["de", "at"]); + expect(screen.getAllByText(/^\d+\.\d\d € · \+\d+\.\d kWh$/)).toHaveLength(1); + expect(screen.getAllByText(/^\+\d+\.\d kWh$/)).toHaveLength(1); + }); + + it("still drops chargers priced in another currency or above the price filter", async () => { + const chargers = [ + makeStation("ok", { price: 0.59, fuelType: "EV", routeFraction: 0.3, detourMin: 0 }), + makeStation("chf", { price: 0.6, currency: "CHF", fuelType: "EV", routeFraction: 0.3, detourMin: 0 }), + makeStation("dear", { price: 0.99, fuelType: "EV", routeFraction: 0.3, detourMin: 0 }), + ]; + renderPlanner({ mode: "ev", stations: chargers, routeKm: 300, maxPrice: 0.8 }); + await userEvent.click(screen.getByText("planner.titleEv")); + expect(planSpy.mock.lastCall![0].stations.map((s: { id: string }) => s.id)).toEqual(["ok"]); + }); + it("converts the session fee from the tariff's currency", async () => { Object.assign(currencyState, { currency: "HUF", symbol: "Ft", decimals: 0, rate: 400, rates: { base: "EUR", rates: { HUF: 400 }, date: "2026-10-22" } }); const chargers = [ diff --git a/src/components/search/refuel-planner.tsx b/src/components/search/refuel-planner.tsx index b1362d3..6b35341 100644 --- a/src/components/search/refuel-planner.tsx +++ b/src/components/search/refuel-planner.tsx @@ -100,8 +100,9 @@ export function RefuelPlanner({ const [timeDraft, setTimeDraft] = useState(null); // EV plans use money only when some charger on the route has a price in the - // display currency; then only priced chargers compete. Otherwise (no feed - // prices the chargers there) they compete on detour and charging time only. + // display currency. Chargers without a price still compete, at 0: most + // countries publish none, and dropping them could leave a cross-border route + // with no plan. With no priced charger at all, only detour and charging time count. const evPriced = ev && stations.some((s) => s.properties.price != null && s.properties.price > 0 && s.properties.currency === currency); const usesMoney = !ev || evPriced; @@ -120,9 +121,9 @@ export function RefuelPlanner({ // Usable stations exist but every detour request failed: filters are not the cause. const detoursFailed = useMemo(() => { - const usable = ev && !evPriced ? stations : stations.filter((s) => s.properties.price != null); + const usable = ev ? stations : stations.filter((s) => s.properties.price != null); return usable.length > 0 && usable.every((s) => s.properties.detourMin != null && s.properties.detourMin < 0); - }, [ev, evPriced, stations]); + }, [ev, stations]); const plan: PlanResult | null = useMemo(() => { if (!open || ratesMissing || detoursPending || routeKm <= 0) return null; @@ -133,8 +134,8 @@ export function RefuelPlanner({ // A power filter can't vouch for a charger whose power is unknown. if (minChargerKw > 0 && !((p.powerKw ?? 0) >= minChargerKw)) return []; const base = { id: p.id, km: (p.routeFraction ?? 0) * routeKm, detourMin: p.detourMin, minutesPerPct: minutesPerPct(evProfile, p.powerKw) }; - if (!evPriced) return [{ ...base, price: 0 }]; - if (p.price == null || p.price <= 0 || p.currency !== currency || (maxPrice != null && p.price > maxPrice)) return []; + if (!evPriced || p.price == null || p.price <= 0) return [{ ...base, price: 0 }]; + if (p.currency !== currency || (maxPrice != null && p.price > maxPrice)) return []; // The session fee is in the tariff's own currency; the price is already converted. const fee = p.evFees?.sessionFee; return [{ ...base, price: p.price, ...(fee != null ? { stopFee: convert(fee, p.originalCurrency ?? p.currency) } : {}) }]; @@ -355,7 +356,7 @@ export function RefuelPlanner({ {ev ? ( <>

    - {evPriced ? `${money(stop.cost)} · ` : ""}+{kwh(stop.litres)} + {evPriced && (f.properties.price ?? 0) > 0 ? `${money(stop.cost)} · ` : ""}+{kwh(stop.litres)}

    {minutes(stop.chargeMin)} · {f.properties.powerKw != null ? `${f.properties.powerKw} kW` : t("popup.powerUnknown")} From 2b8535d5608c7404d4585647e43515fda4056f8c Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:59:04 +0200 Subject: [PATCH 14/25] fix(ocpdb): merge register sites only at 5 dp --- src/scrapers/ocpdb.test.ts | 15 +++++++++++++++ src/scrapers/ocpdb.ts | 22 +++++++++++++++------- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/scrapers/ocpdb.test.ts b/src/scrapers/ocpdb.test.ts index 2144c34..bdb9597 100644 --- a/src/scrapers/ocpdb.test.ts +++ b/src/scrapers/ocpdb.test.ts @@ -417,6 +417,21 @@ describe("mergeLocations", () => { expect(prices[0].reportedAt).toEqual(refreshed); }); + it("keeps register sites a few metres apart separate", () => { + const { stations, stats } = mergeLocations( + [location(), location({ lat: 48.7758 + 10 * METRES, operator: { name: "Stadtwerke Stuttgart" } })], + tariffs, + ); + expect(stats.merged).toBe(0); + expect(stations.map((s) => s.brand)).toEqual(["EnBW mobility+ AG und Co.KG", "Stadtwerke Stuttgart"]); + }); + + it("merges register rows that share a site at 5 dp", () => { + const { stations, stats } = mergeLocations([location(), location({ lat: 48.775801 })], tariffs); + expect(stats.merged).toBe(1); + expect(stations).toHaveLength(1); + }); + it("keeps locations further apart than the merge radius separate", () => { const { stations } = mergeLocations([location(), location({ lat: 48.7758 + 40 * METRES })], tariffs); expect(stations).toHaveLength(2); diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts index 429ac0f..0f0f03e 100644 --- a/src/scrapers/ocpdb.ts +++ b/src/scrapers/ocpdb.ts @@ -32,7 +32,10 @@ import { otherGermanyEvPrefixes } from "./germany-ev-source"; // CPO) and with coordinates a few metres off. Measured on 2026-10-22: // ~4.7k EcoMovement and ~4.9k Chargecloud locations sit within 25 m of a // register location, many at 0 m. Hence mergeLocations(): a 25 m greedy -// merge, register first so its coordinates anchor the pins. +// merge of feeds onto the register, register first so its coordinates +// anchor the pins. Register sites only merge with each other at 5 dp, as +// in bnetza.ts: 25 m would fold ~9k distinct register sites together, +// ~850 of them run by another operator. // // 2. VAT is inconsistent across feeds. EcoMovement and EnBW publish net // prices with `taxes: [VAT 19%]` and `tax_included: false`, Wirelane @@ -62,7 +65,7 @@ const PAGE_TIMEOUT_MS = 120_000; // Germany's bounding box, padded. Same as bnetza.ts. const DE_BBOX = { latMin: 47, latMax: 56, lonMin: 5.5, lonMax: 15.5 }; -/** Locations closer than this fold into one pin. */ +/** Feed locations closer than this to a pin fold into it. */ const MERGE_RADIUS_M = 25; // Grid cell for the merge lookup, in degrees. ~111 m of latitude, so a 3×3 // neighbourhood always covers the merge radius. @@ -71,12 +74,14 @@ const GRID_DEG = 0.001; // Baked into every externalId. Changing it re-mints every row. const COORD_PRECISION = 5; +const REGISTER_SOURCE = "bnetza_api"; + // Lower rank wins the anchor. The register first: its coordinates are the // official ones and it covers every operator. Then the feeds that carry // prices, so a priced pin keeps the operator's own name when the register has // no entry nearby. const SOURCE_RANK: Record = { - bnetza_api: 0, + [REGISTER_SOURCE]: 0, datex2_enbw: 1, datex2_ecomovement: 2, datex2_chargecloud: 3, @@ -413,6 +418,7 @@ export function mergeLocations( .map((x) => x.loc); const grid = new Map(); + const byKey = new Map(); const pins: Pin[] = []; const cell = (lat: number, lon: number) => `${Math.floor(lat / GRID_DEG)}:${Math.floor(lon / GRID_DEG)}`; @@ -451,12 +457,14 @@ export function mergeLocations( if (powerKw != null && evsePower[i] === powerKw) top = cheaperOffer(offer, top); } - // Fold into an existing pin within the merge radius, if any. + // Fold into an existing pin: a register site only into one at the same + // 5 dp key, a feed into the nearest within the merge radius. + const key = stationKey(latitude, longitude); const gy = Math.floor(latitude / GRID_DEG); const gx = Math.floor(longitude / GRID_DEG); - let target: Pin | null = null; + let target: Pin | null = loc.source === REGISTER_SOURCE ? (byKey.get(key) ?? null) : null; let best = MERGE_RADIUS_M; - for (let dy = -1; dy <= 1; dy++) { + for (let dy = -1; dy <= 1 && loc.source !== REGISTER_SOURCE; dy++) { for (let dx = -1; dx <= 1; dx++) { for (const pin of grid.get(`${gy + dy}:${gx + dx}`) ?? []) { const d = distanceM(pin.station.latitude, pin.station.longitude, latitude, longitude); @@ -482,7 +490,6 @@ export function mergeLocations( continue; } - const key = stationKey(latitude, longitude); const street = loc.address?.trim() ?? ""; const postcode = loc.postal_code?.trim() ?? ""; const site = loc.name?.trim() || street; @@ -505,6 +512,7 @@ export function mergeLocations( site, }; pins.push(pin); + byKey.set(key, pin); const k = cell(latitude, longitude); const bucket = grid.get(k); if (bucket) bucket.push(pin); From 0793e194b9e3aa9b802460851408abae15fc8f59 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:59:56 +0200 Subject: [PATCH 15/25] fix(ocpdb): drop implausible per-minute fees and show tiny ones per hour --- src/components/map/station-popup.test.tsx | 7 +++++++ src/components/map/station-popup.tsx | 5 ++++- src/scrapers/ocpdb.test.ts | 23 +++++++++++++++++++++++ src/scrapers/ocpdb.ts | 14 ++++++++++++-- 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/components/map/station-popup.test.tsx b/src/components/map/station-popup.test.tsx index 724d87e..976e8ec 100644 --- a/src/components/map/station-popup.test.tsx +++ b/src/components/map/station-popup.test.tsx @@ -158,6 +158,13 @@ describe("StationPopup", () => { expect(screen.getByText("+0.10 €/min")).toBeInTheDocument(); }); + it("shows a per-minute fee under a cent per hour", () => { + render( + {}} />, + ); + expect(screen.getByText("+0.18 €/h")).toBeInTheDocument(); + }); + it("lists every option of a session fee that varies", () => { render( { }); }); + // Chargecloud 133462/133483/133484: the kWh rate repeated as an always-on + // TIME component, ahead of the real blocking fee. + it("ignores a TIME price that repeats the energy price", () => { + const tariff: Tariff = { + ...CHARGECLOUD, + elements: [ + { price_components: [{ type: "ENERGY", price: 0.56 }] }, + { price_components: [{ type: "TIME", price: 0.56 }] }, + { price_components: [{ type: "TIME", price: 0.05 }], restrictions: { min_duration: 14400 } }, + ], + }; + expect(summarizeTariff(tariff)?.fees).toEqual({ perMinute: 0.05, afterMin: 240 }); + }); + + // EcoMovement 133247. + it("ignores an implausible per-minute fee", () => { + const tariff: Tariff = { + ...CHARGECLOUD, + elements: [{ price_components: [{ type: "ENERGY", price: 0.59 }, { type: "TIME", price: 14.28 }] }], + }; + expect(summarizeTariff(tariff)?.fees).toEqual({}); + }); + it("is null without a per-kWh price", () => { const t: Tariff = { id: "1", source: "x", currency: "EUR", elements: [{ price_components: [{ type: "FLAT", price: 5 }] }] }; expect(summarizeTariff(t)).toBeNull(); diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts index 0f0f03e..a583fdf 100644 --- a/src/scrapers/ocpdb.ts +++ b/src/scrapers/ocpdb.ts @@ -49,7 +49,10 @@ import { otherGermanyEvPrefixes } from "./germany-ev-source"; // one seen is a blocking fee starting 30-240 min in (EnBW 0.2017 net // after 30 min, Wirelane 0.20 after 45 min, Chargecloud 0.06 after // 4 h). Those match the per-minute blocking fees operators advertise. -// Per hour they would be too small to block anyone. +// Per hour they would be too small to block anyone. Two feed errors are +// dropped: a TIME price equal to the tariff's ENERGY price (Chargecloud +// repeats the kWh rate as an always-on TIME component), and anything +// above MAX_PER_MINUTE (EcoMovement 133247 publishes 14.28/min). // // Like bnetza.ts, run() sweeps afterwards, because base.run() never // garbage-collects EV rows. @@ -95,6 +98,10 @@ const DEAD_EVSE_STATUSES = new Set(["REMOVED", "PLANNED"]); // German standard VAT, for prices marked net that don't say at what rate. const DE_VAT_PCT = 19; +// Highest believable per-minute fee. Blocking fees seen run 0.05-0.24/min, +// so this leaves wide headroom while catching unit errors like 14.28/min. +const MAX_PER_MINUTE = 1; + // Session size used to fold a session fee into a per-kWh figure when ranking // tariffs: a typical fast-charge top-up. const REF_SESSION_KWH = 30; @@ -242,6 +249,9 @@ export function summarizeTariff(tariff: OcpdbTariff): TariffSummary | null { let sessionFee = 0; const rules: SessionFeeRule[] = []; let perMinute: { price: number; afterSec: number } | null = null; + const energyPrices = new Set( + tariff.elements.flatMap((e) => e.price_components.filter((c) => c.type === "ENERGY").map((c) => grossPrice(c, tariff))), + ); for (const element of tariff.elements) { const afterSec = element.restrictions?.min_duration ?? 0; @@ -255,7 +265,7 @@ export function summarizeTariff(tariff: OcpdbTariff): TariffSummary | null { } else if (c.type === "FLAT") { if (!restricted) sessionFee += price; else rules.push(feeRule(price, element.restrictions ?? {})); - } else if (c.type === "TIME" && price > 0) { + } else if (c.type === "TIME" && price > 0 && price <= MAX_PER_MINUTE && !energyPrices.has(price)) { // Several TIME steps: the first one that costs anything is the fee // a driver will actually meet. if (perMinute == null || afterSec < perMinute.afterSec) perMinute = { price, afterSec }; From f4a5f2095a369875455c0c107e2f3a2b7fb3531e Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:00:22 +0200 Subject: [PATCH 16/25] perf(ocpdb): parse location pages as they arrive --- src/scrapers/ocpdb.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts index a583fdf..9126632 100644 --- a/src/scrapers/ocpdb.ts +++ b/src/scrapers/ocpdb.ts @@ -587,24 +587,26 @@ async function fetchPage(path: string, offset: number): Promise(path: string, schema: z.ZodType): Promise<{ items: T[]; invalid: number }> { - const first = await fetchPage(path, 0); - const offsets: number[] = []; - for (let o = PAGE_SIZE; o < first.total_count; o += PAGE_SIZE) offsets.push(o); - - const pages: unknown[][] = [first.items]; - for (let i = 0; i < offsets.length; i += CONCURRENCY) { - const batch = await Promise.all(offsets.slice(i, i + CONCURRENCY).map((o) => fetchPage(path, o))); - for (const p of batch) pages.push(p.items); - } - const items: T[] = []; let invalid = 0; - for (const page of pages) { + // Parse each page as it lands and let the raw JSON go: holding all ~100 + // location pages until the end peaked at ~800 MB RSS. + const keep = (page: unknown[]) => { for (const raw of page) { const r = schema.safeParse(raw); if (r.success) items.push(r.data); else invalid++; } + }; + + const first = await fetchPage(path, 0); + keep(first.items); + const offsets: number[] = []; + for (let o = PAGE_SIZE; o < first.total_count; o += PAGE_SIZE) offsets.push(o); + + for (let i = 0; i < offsets.length; i += CONCURRENCY) { + const batch = await Promise.all(offsets.slice(i, i + CONCURRENCY).map((o) => fetchPage(path, o))); + for (const p of batch) keep(p.items); } return { items, invalid }; } From 3dc42bbc49d431e1871a46a13eb9cc0ff7f69947 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:00:49 +0200 Subject: [PATCH 17/25] fix(ocpdb): leave tariffs with conflicting energy prices unpriced --- src/scrapers/ocpdb.test.ts | 25 +++++++++++++++++++++++++ src/scrapers/ocpdb.ts | 8 ++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/scrapers/ocpdb.test.ts b/src/scrapers/ocpdb.test.ts index cbd269f..2443bb2 100644 --- a/src/scrapers/ocpdb.test.ts +++ b/src/scrapers/ocpdb.test.ts @@ -194,6 +194,31 @@ describe("summarizeTariff", () => { expect(summarizeTariff(tariff)?.fees).toEqual({}); }); + // ChargePoint 132894. + it("is null when unrestricted energy prices disagree", () => { + const tariff: Tariff = { + ...CHARGECLOUD, + elements: [ + { price_components: [{ type: "ENERGY", price: 1.6 }] }, + { price_components: [{ type: "ENERGY", price: 0.412 }] }, + { price_components: [{ type: "ENERGY", price: 0.16 }] }, + ], + }; + expect(summarizeTariff(tariff)).toBeNull(); + }); + + it("keeps a tariff whose extra energy prices are restricted or repeat the first", () => { + const tariff: Tariff = { + ...CHARGECLOUD, + elements: [ + { price_components: [{ type: "ENERGY", price: 0.49 }] }, + { price_components: [{ type: "ENERGY", price: 0.49 }] }, + { price_components: [{ type: "ENERGY", price: 0.39 }], restrictions: { start_time: "22:00", end_time: "06:00" } }, + ], + }; + expect(summarizeTariff(tariff)?.energy).toBe(0.49); + }); + it("is null without a per-kWh price", () => { const t: Tariff = { id: "1", source: "x", currency: "EUR", elements: [{ price_components: [{ type: "FLAT", price: 5 }] }] }; expect(summarizeTariff(t)).toBeNull(); diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts index 9126632..ec5bcd8 100644 --- a/src/scrapers/ocpdb.ts +++ b/src/scrapers/ocpdb.ts @@ -239,7 +239,7 @@ function cheaper(a: TariffSummary, b: TariffSummary | null | undefined): boolean /** * Reduce a tariff to what a pin can show: gross €/kWh plus the fees on top. * Null when the tariff has no per-kWh price (a flat- or time-only tariff - * cannot be ranked against the rest). + * cannot be ranked against the rest), or more than one unrestricted one. */ export function summarizeTariff(tariff: OcpdbTariff): TariffSummary | null { let energy: number | null = null; @@ -249,6 +249,7 @@ export function summarizeTariff(tariff: OcpdbTariff): TariffSummary | null { let sessionFee = 0; const rules: SessionFeeRule[] = []; let perMinute: { price: number; afterSec: number } | null = null; + const unrestrictedEnergy = new Set(); const energyPrices = new Set( tariff.elements.flatMap((e) => e.price_components.filter((c) => c.type === "ENERGY").map((c) => grossPrice(c, tariff))), ); @@ -262,6 +263,7 @@ export function summarizeTariff(tariff: OcpdbTariff): TariffSummary | null { // Restricted energy prices (by duration, power...) are the exception; // the first one listed is the headline rate. if (energy == null) energy = price; + if (!restricted) unrestrictedEnergy.add(price); } else if (c.type === "FLAT") { if (!restricted) sessionFee += price; else rules.push(feeRule(price, element.restrictions ?? {})); @@ -273,7 +275,9 @@ export function summarizeTariff(tariff: OcpdbTariff): TariffSummary | null { } } - if (energy == null) return null; + // Several always-on kWh rates that disagree (132894 lists 1.6, 0.412 and + // 0.16 net) can't say which one a driver pays: leave the tariff unpriced. + if (energy == null || unrestrictedEnergy.size > 1) return null; const fees: EvFees = {}; const highest = Math.max(sessionFee, ...rules.map((r) => r.fee)); if (highest > 0) { From 7c35f2cfed55a8ee2c898591845490b21025fd75 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:02:17 +0200 Subject: [PATCH 18/25] docs: credit OCPDB under dl-de/by-2-0 and note the rollback cleanup --- docs/data/ev-sources.md | 15 +++++++++++++-- docs/data/sources-at-a-glance.md | 2 +- src/components/nav/legal-modal.tsx | 2 +- src/scrapers/ocpdb.ts | 4 +++- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/data/ev-sources.md b/docs/data/ev-sources.md index 0bc7cf6..eb349f7 100644 --- a/docs/data/ev-sources.md +++ b/docs/data/ev-sources.md @@ -8,7 +8,7 @@ This page describes where Pumperly's EV charging locations come from, how each s |---|---|---|---|---| | [Open Charge Map](#open-charge-map) | Every supported country and the United States | `PUMPERLY_OCM_API_KEY` | 24 h | Open Database License (ODbL) | | [Mapa REVE](#mapa-reve) | Spain | `PUMPERLY_REVE_API_KEY` | 1 h | Non-commercial use, credit Red Eléctrica de España | -| [Open ChargePoint Database (OCPDB)](#ocpdb) | Germany, with prices | None | 6 h | CC BY 4.0 for the register and EnBW, CC0 for the other feeds | +| [Open ChargePoint Database (OCPDB)](#ocpdb) | Germany, with prices | None | 6 h | dl-de/by-2-0 (MobiData BW); the register and EnBW are CC BY 4.0 | | [BNetzA Ladesäulenregister](#bnetza) | Germany | None | 24 h | CC BY 4.0, credit "Bundesnetzagentur.de" | Open Charge Map is crowdsourced: anyone can add or edit a charger. Mapa REVE and the BNetzA register are official registries that charge point operators must file into, so they are authoritative where Open Charge Map is crowdsourced. OCPDB contains the BNetzA register and adds the operators' own feeds, which carry ad-hoc tariffs. @@ -214,7 +214,7 @@ The [Open ChargePoint Database](https://api.mobidata-bw.de/ocpdb/) is run by Mob | Station ids | `ocpdb-_` | | Every | 6 h | | API key | None | -| Licence | The register and EnBW are CC BY 4.0, the other feeds CC0. The attribution is in the app's legal notice. | +| Licence | The bundle is [dl-de/by-2-0](https://www.govdata.de/dl-de/by-2-0), credited to MobiData BW. Within it the register and EnBW are CC BY 4.0. Some feeds declare no licence of their own, so the bundle's licence is the one that covers them. The attribution is in the app's legal notice. | ### How the data is read {#ocpdb-reading} @@ -317,6 +317,8 @@ The floor counts only rows this run wrote, not rows already in the database. A b Open Charge Map deletes nothing when it takes over. Official sources delete Open Charge Map's rows, and OCPDB and BNetzA delete each other's, but nothing deletes the official rows once their scraper stops running and Open Charge Map runs instead. If you switch Spain or Germany back to Open Charge Map, the official pins stay, and Open Charge Map adds its own next to them. +Rolling back to a Pumperly release from before OCPDB has the same effect in Germany. That release's BNetzA scraper does not know the `ocpdb-` rows, so it leaves them in place and adds its own pins next to them, and every German charger shows twice. + To clear the old rows after switching back, delete them in `psql` against the Pumperly database: === "Spain, after removing the REVE key" @@ -337,4 +339,13 @@ To clear the old rows after switching back, delete them in `psql` against the Pu AND (external_id LIKE 'ocpdb-%' OR external_id LIKE 'bnetza-%'); ``` +=== "Germany, after rolling back to a release without OCPDB" + + ```sql + DELETE FROM stations + WHERE country = 'DE' + AND station_type = 'ev_charger' + AND external_id LIKE 'ocpdb-%'; + ``` + A charger's prices are deleted with it, so nothing else needs deleting. [Backing up the database](../operations/backup-and-restore.md) explains how to take a backup first. diff --git a/docs/data/sources-at-a-glance.md b/docs/data/sources-at-a-glance.md index eaee61b..902b716 100644 --- a/docs/data/sources-at-a-glance.md +++ b/docs/data/sources-at-a-glance.md @@ -42,7 +42,7 @@ One table per kind of source. The pages under Data sources explain each source i |---|---|---| | [Open Charge Map](https://openchargemap.org) | All supported countries + United States (EV-only) | ODbL | | [Mapa REVE](https://www.mapareve.es) (Red Eléctrica de España) | Spain — official operator-reported registry | Non-commercial, attribution required | -| [Open ChargePoint Database](https://api.mobidata-bw.de/ocpdb/) (MobiData BW) | Germany — the Ladesäulenregister merged with operator feeds, with ad-hoc prices | CC BY 4.0 for the register and EnBW, CC0 for the other feeds | +| [Open ChargePoint Database](https://api.mobidata-bw.de/ocpdb/) (MobiData BW) | Germany — the Ladesäulenregister merged with operator feeds, with ad-hoc prices | dl-de/by-2-0 (MobiData BW); the register and EnBW are CC BY 4.0 | | [Ladesäulenregister](https://www.bundesnetzagentur.de/DE/Fachthemen/ElektrizitaetundGas/E-Mobilitaet/Ladesaeulenkarte/start.html) (Bundesnetzagentur) | Germany — official operator-reported registry, ~74,000 locations | CC BY 4.0, attribution "Bundesnetzagentur.de" | Spain uses Mapa REVE when `PUMPERLY_REVE_API_KEY` is set: it is the registry every Spanish charge point operator files into, so it is authoritative where Open Charge Map is crowdsourced. The two overlap heavily, so Open Charge Map stops being scraped for Spain immediately, its existing Spanish rows stay visible while REVE backfills, and they are deleted once REVE reaches 95% of the registry. Expect duplicate Spanish pins until then. diff --git a/src/components/nav/legal-modal.tsx b/src/components/nav/legal-modal.tsx index a024055..9fc8ff0 100644 --- a/src/components/nav/legal-modal.tsx +++ b/src/components/nav/legal-modal.tsx @@ -164,7 +164,7 @@ function SourcesContent() {

  • Open Charge Map — EV charging station locations across all supported countries. Community-maintained, Open Data Commons Open Database License (ODbL). openchargemap.org
  • Mapa REVE — EV charging station locations in Spain. Source: Red Eléctrica de España, S.A.U. Used non-commercially and reproduced without alteration. mapareve.es
  • Ladesäulenregister — EV charging station locations in Germany, from the register operators must file under §5 Ladesäulenverordnung. Source: Bundesnetzagentur.de, Creative Commons Attribution 4.0 (CC BY 4.0). bundesnetzagentur.de
    • -
  • Open ChargePoint Database (OCPDB) — EV charging station locations and ad-hoc prices in Germany, served by MobiData BW (NVBW). Combines the Ladesäulenregister (Bundesnetzagentur.de, CC BY 4.0) with operator feeds published on Mobilithek: EnBW AG (CC BY 4.0); EcoMovement, Chargecloud, Wirelane, e-clearing.net, Tesla and others (CC0). mobidata-bw.de
    • +
  • Open ChargePoint Database (OCPDB) — EV charging station locations and ad-hoc prices in Germany, served by MobiData BW (NVBW). Source: MobiData BW, Datenlizenz Deutschland – Namensnennung – Version 2.0 (dl-de/by-2-0). Combines the Ladesäulenregister (Bundesnetzagentur.de, CC BY 4.0) with operator feeds published on Mobilithek, among them EnBW AG (CC BY 4.0), EcoMovement, Chargecloud, Wirelane, e-clearing.net and Tesla. mobidata-bw.de

    • Map and routing

    diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts index ec5bcd8..5ba8447 100644 --- a/src/scrapers/ocpdb.ts +++ b/src/scrapers/ocpdb.ts @@ -19,7 +19,9 @@ import { otherGermanyEvPrefixes } from "./germany-ev-source"; // DATEX II feeds from Mobilithek (EnBW, EcoMovement, Chargecloud, // Wirelane, e-clearing.net, Tesla, ...). Freshness per source: // /ocpdb/api/public/v1/sources -// Licence: BNetzA and EnBW are CC BY 4.0, the other feeds CC0 (see +// Licence: the bundle is dl-de/by-2-0 (MobiData BW); within it BNetzA and +// EnBW are CC BY 4.0. Not every feed declares a licence of its own, +// so the bundle's is the one credited (see // components/nav/legal-modal.tsx). No API key, no signup. // // Replaces the BNetzA scraper as Germany's default EV source: same register, From 6e9a9c4f0d30615043cd3e9e35281aa7ea4673b1 Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:39:07 +0200 Subject: [PATCH 19/25] fix(planner): plan EV by time when the filters leave no priced charger --- src/components/search/refuel-planner.test.tsx | 16 +++++++++++++ src/components/search/refuel-planner.tsx | 23 ++++++++++++------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/components/search/refuel-planner.test.tsx b/src/components/search/refuel-planner.test.tsx index 03fb60e..f81c4a1 100644 --- a/src/components/search/refuel-planner.test.tsx +++ b/src/components/search/refuel-planner.test.tsx @@ -462,6 +462,22 @@ describe("RefuelPlanner", () => { expect(planSpy.mock.lastCall![0].stations.map((s: { id: string }) => s.id)).toEqual(["ok"]); }); + it("plans by time only when the filters leave no priced charger", async () => { + const chargers = [ + makeStation("dear", { price: 0.99, fuelType: "EV", routeFraction: 0.3, detourMin: 0, powerKw: 150 }), + makeStation("slow", { price: 0.49, fuelType: "EV", routeFraction: 0.3, detourMin: 0, powerKw: 22 }), + makeStation("free", { price: null, currency: undefined, fuelType: "EV", routeFraction: 0.6, detourMin: 0, powerKw: 150 }), + ]; + renderPlanner({ mode: "ev", stations: chargers, routeKm: 300, maxPrice: 0.8 }); + await userEvent.click(screen.getByText("planner.titleEv")); + expect(planSpy.mock.lastCall![0].timeValuePerHour).toBe(15); + await userEvent.selectOptions(screen.getByLabelText("planner.minChargerKw"), "50"); + const input = planSpy.mock.lastCall![0]; + expect(input.stations.map((s: { id: string }) => s.id)).toEqual(["free"]); + expect(input.timeValuePerHour).toBe(1); + expect(screen.queryByText(/planner.timeValue/)).not.toBeInTheDocument(); + }); + it("converts the session fee from the tariff's currency", async () => { Object.assign(currencyState, { currency: "HUF", symbol: "Ft", decimals: 0, rate: 400, rates: { base: "EUR", rates: { HUF: 400 }, date: "2026-10-22" } }); const chargers = [ diff --git a/src/components/search/refuel-planner.tsx b/src/components/search/refuel-planner.tsx index 6b35341..82c95a2 100644 --- a/src/components/search/refuel-planner.tsx +++ b/src/components/search/refuel-planner.tsx @@ -18,6 +18,13 @@ const EV_MAX_CHARGE_PCT = 80; /** "Min charger power" choices, kW; 0 = any (including chargers with unknown power). */ const MIN_CHARGER_KW_OPTIONS = [0, 50, 150] as const; +/** Whether a charger passes the detour and power filters; a detour still loading passes. */ +function evEligible(p: StationGeoJSON["properties"], maxDetour: number | null | undefined, minChargerKw: number): boolean { + if (p.detourMin != null && (p.detourMin < 0 || (maxDetour != null && p.detourMin > maxDetour))) return false; + // A power filter can't vouch for a charger whose power is unknown. + return !(minChargerKw > 0 && !((p.powerKw ?? 0) >= minChargerKw)); +} + interface RefuelPlannerProps { /** `ev` plans in kWh, by price when chargers on the route have one, else by time only. */ mode?: "fuel" | "ev"; @@ -99,11 +106,12 @@ export function RefuelPlanner({ // Same for the value of time; null shows the saved (and planned) value. const [timeDraft, setTimeDraft] = useState(null); - // EV plans use money only when some charger on the route has a price in the - // display currency. Chargers without a price still compete, at 0: most + // EV plans use money only when some charger the filters leave has a price in + // the display currency. Chargers without a price still compete, at 0: most // countries publish none, and dropping them could leave a cross-border route - // with no plan. With no priced charger at all, only detour and charging time count. - const evPriced = ev && stations.some((s) => s.properties.price != null && s.properties.price > 0 && s.properties.currency === currency); + // with no plan. With no such priced charger, only detour and charging time count. + const evPriced = ev && stations.some(({ properties: p }) => + evEligible(p, maxDetour, minChargerKw) && p.price != null && p.price > 0 && p.currency === currency && (maxPrice == null || p.price <= maxPrice)); const usesMoney = !ev || evPriced; // The time value is stored in EUR: without a rate for the display currency it @@ -130,12 +138,11 @@ export function RefuelPlanner({ const candidates = stations.flatMap((s) => { const p = s.properties; if (ev) { - if (p.detourMin == null || p.detourMin < 0 || (maxDetour != null && p.detourMin > maxDetour)) return []; - // A power filter can't vouch for a charger whose power is unknown. - if (minChargerKw > 0 && !((p.powerKw ?? 0) >= minChargerKw)) return []; + if (p.detourMin == null || !evEligible(p, maxDetour, minChargerKw)) return []; + if (maxPrice != null && p.price != null && p.price > maxPrice && p.currency === currency) return []; const base = { id: p.id, km: (p.routeFraction ?? 0) * routeKm, detourMin: p.detourMin, minutesPerPct: minutesPerPct(evProfile, p.powerKw) }; if (!evPriced || p.price == null || p.price <= 0) return [{ ...base, price: 0 }]; - if (p.currency !== currency || (maxPrice != null && p.price > maxPrice)) return []; + if (p.currency !== currency) return []; // The session fee is in the tariff's own currency; the price is already converted. const fee = p.evFees?.sessionFee; return [{ ...base, price: p.price, ...(fee != null ? { stopFee: convert(fee, p.originalCurrency ?? p.currency) } : {}) }]; From 9c90da4db87298d1c36146737b7544df3136210f Mon Sep 17 00:00:00 2001 From: yellowhat <1692490+yellowhat@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:41:24 +0200 Subject: [PATCH 20/25] fix(ocpdb): fail the run on unparseable records instead of dropping them --- src/scrapers/ocpdb.test.ts | 24 +++++++++++++----------- src/scrapers/ocpdb.ts | 7 +++++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/scrapers/ocpdb.test.ts b/src/scrapers/ocpdb.test.ts index 2443bb2..52e07b7 100644 --- a/src/scrapers/ocpdb.test.ts +++ b/src/scrapers/ocpdb.test.ts @@ -518,7 +518,14 @@ describe("OcpdbScraper.fetch", () => { /** Routes each URL to a page; records every URL requested. */ function stubApi( - opts: { sources?: unknown[]; tariffs?: unknown[]; locationsTotal?: number; failOnce?: string; failAlways?: string } = {}, + opts: { + sources?: unknown[]; + tariffs?: unknown[]; + locationsTotal?: number; + failOnce?: string; + failAlways?: string; + broken?: boolean; + } = {}, ) { const urls: string[] = []; const failed = new Set(); @@ -551,9 +558,9 @@ describe("OcpdbScraper.fetch", () => { } const offset = Number(new URL(url).searchParams.get("offset")); // One location per page, a degree of latitude apart so none merge. - // The first page also carries one record the schema rejects. + // With `broken`, the first page also carries one record the schema rejects. const items: unknown[] = [location({ lat: 48 + offset / 1000, evses: [offset === 0 ? "enbw" : "cc"] })]; - if (offset === 0) items.push({ source: "broken" }); + if (offset === 0 && opts.broken) items.push({ source: "broken" }); return page(items, opts.locationsTotal ?? 1); }), ); @@ -572,16 +579,11 @@ describe("OcpdbScraper.fetch", () => { expect(stations).toHaveLength(3); }); - it("skips records that fail to parse instead of failing the run", async () => { - stubApi(); + it("fails the run on a record that does not parse, so the current prices stay", async () => { + stubApi({ broken: true }); const { OcpdbScraper } = await import("./ocpdb"); - vi.spyOn(console, "log").mockImplementation(() => {}); - - const { stations, prices } = await new OcpdbScraper().fetch(); - expect(stations).toHaveLength(1); - expect(prices).toEqual([expect.objectContaining({ fuelType: "EV", price: 0.79 })]); - expect(vi.mocked(console.log).mock.calls.flat().join(" ")).toMatch(/1 unparseable/); + await expect(new OcpdbScraper().fetch()).rejects.toThrow(/1 unparseable OCPDB records/); }); // TotalEnergies tariffs on EcoMovement (132622, 132721, 132724, 133685) give diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts index 5ba8447..e802916 100644 --- a/src/scrapers/ocpdb.ts +++ b/src/scrapers/ocpdb.ts @@ -651,6 +651,10 @@ export class OcpdbScraper extends BaseScraper { fetchAll("tariff-associations", AssociationSchema), fetchAll("locations?country=DEU", LocationSchema), ]); + // A dropped tariff or association would silently unprice its stations, and + // a dropped location would be swept: keep the current rows instead. + const invalid = tariffs.invalid + associations.invalid + locations.invalid; + if (invalid > 0) throw new Error(`[${this.source}] DE: ${invalid} unparseable OCPDB records`); const live = liveSources(sources, new Date(), MAX_FEED_AGE_DAYS); const stale = [...new Set(tariffs.items.map((t) => t.source))].filter((s) => !live.has(s)); if (stale.length > 0) { @@ -665,8 +669,7 @@ export class OcpdbScraper extends BaseScraper { console.log( `[${this.source}] DE: ${tariffs.items.length} tariffs → ${tariffsByEvse.size} priced EVSEs; ` + `${stats.locations} locations → ${stations.length} stations, ${stats.priced} priced ` + - `(merged ${stats.merged}, skipped ${stats.noLiveEvse} without live EVSE, ${stats.notGermany} outside DE, ` + - `${tariffs.invalid + associations.invalid + locations.invalid} unparseable)`, + `(merged ${stats.merged}, skipped ${stats.noLiveEvse} without live EVSE, ${stats.notGermany} outside DE)`, ); return { stations, prices }; From 1493430463afc2f997e7fa88fcfbcee6618f39b2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:17:08 +0200 Subject: [PATCH 21/25] fix(planner): cost unpriced chargers at the median price, not as free Unpriced chargers went into a priced EV plan at 0 per kWh, so the planner treated them as free. In Germany about three in four OCPDB pins have no price, so plans picked them wherever they could and the total came out far too low. They now compete at the median price of the priced chargers that pass the filters. Their stop shows kWh only, and the plan shows a total only when every stop has a real price. --- docs/using/ev-charging.md | 2 +- src/components/search/refuel-planner.test.tsx | 11 +++++-- src/components/search/refuel-planner.tsx | 30 ++++++++++++++----- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/docs/using/ev-charging.md b/docs/using/ev-charging.md index bd5a538..6da507d 100644 --- a/docs/using/ev-charging.md +++ b/docs/using/ev-charging.md @@ -81,7 +81,7 @@ Clicking a charger in the list or on the map adds it to the route as a stop, as On the EV layer, the fuel stop planner becomes **Plan charging stops**. You set the battery size in kWh, the consumption in kWh/100 km, the car's max charge power, and the battery level at the start and at arrival. -When any charger along the route has a price, the planner uses only priced chargers and picks the stops with the lowest cost: the energy at each charger's price per kWh, plus its session fee, plus your time, priced with **Value of your time**. Each stop shows its cost, and so does the plan. A session fee that varies counts at its highest. Per-minute blocking fees are left out. When no charger along the route has a price, as outside Germany, the planner picks the stops with the least detour and charging time. +When any charger along the route that passes the filters has a price, the planner picks the stops with the lowest cost: the energy at each charger's price per kWh, plus its session fee, plus your time, priced with **Value of your time**. Unpriced chargers still compete, costed at the median price of the priced ones, so a route that crosses into a country without prices still gets a plan. Each priced stop shows its cost. The plan shows a total only when every stop has a price. A session fee that varies counts at its highest. Per-minute blocking fees are left out. When no charger along the route has a price, as outside Germany, the planner picks the stops with the least detour and charging time. Each stop charges to at most 80 %, because charging slows sharply above that. Charging time uses the lower of the charger's power and the car's max charge power, capped at 22 kW on AC chargers. On DC it averages 75 % of that, because charging tapers as the battery fills. The reserve defaults to 20 %. **Min charger power** (Any, 50 kW or 150 kW) skips slower chargers. It also skips chargers whose power is unknown, because the filter can't vouch for them. diff --git a/src/components/search/refuel-planner.test.tsx b/src/components/search/refuel-planner.test.tsx index f81c4a1..39d3f6d 100644 --- a/src/components/search/refuel-planner.test.tsx +++ b/src/components/search/refuel-planner.test.tsx @@ -430,25 +430,32 @@ describe("RefuelPlanner", () => { expect(input.timeValuePerHour).toBe(15); expect(screen.getByText(/planner.timeValue/)).toBeInTheDocument(); expect(screen.getAllByText(/^\d+\.\d\d € · \+\d+\.\d kWh$/)).toHaveLength(2); + // Every stop priced: the plan shows its total cost. + expect(screen.getByText(/ · \d+\.\d\d €$/)).toBeInTheDocument(); }); // Munich to Vienna: German chargers carry OCPDB prices, Austrian ones none. - it("keeps unpriced chargers in a priced plan, at 0 and with no cost shown", async () => { + it("keeps unpriced chargers in a priced plan, at the median price and with no cost shown", async () => { const chargers = [ makeStation("de", { brand: "EnBW", price: 0.79, fuelType: "EV", routeFraction: 0.3, detourMin: 0 }), + makeStation("de2", { brand: "Aral", price: 0.59, fuelType: "EV", routeFraction: 0.1, detourMin: 30 }), makeStation("at", { brand: "Smatrics", price: null, currency: undefined, fuelType: "EV", routeFraction: 0.6, detourMin: 0 }), ]; const { onPlanChange } = renderPlanner({ mode: "ev", stations: chargers, routeKm: 300 }); await userEvent.click(screen.getByText("planner.titleEv")); const input = planSpy.mock.lastCall![0]; + // Neither free nor dropped: the median of the priced ones, 0.69. expect(input.stations).toEqual([ expect.objectContaining({ id: "de", price: 0.79 }), - expect.objectContaining({ id: "at", price: 0 }), + expect.objectContaining({ id: "de2", price: 0.59 }), + expect.objectContaining({ id: "at", price: 0.69 }), ]); expect(input.timeValuePerHour).toBe(15); expect(onPlanChange.mock.lastCall![0].map((s: { id: string }) => s.id)).toEqual(["de", "at"]); expect(screen.getAllByText(/^\d+\.\d\d € · \+\d+\.\d kWh$/)).toHaveLength(1); expect(screen.getAllByText(/^\+\d+\.\d kWh$/)).toHaveLength(1); + // An estimated stop: no total cost. + expect(screen.queryByText(/ · \d+\.\d\d €$/)).not.toBeInTheDocument(); }); it("still drops chargers priced in another currency or above the price filter", async () => { diff --git a/src/components/search/refuel-planner.tsx b/src/components/search/refuel-planner.tsx index 82c95a2..9dcda4b 100644 --- a/src/components/search/refuel-planner.tsx +++ b/src/components/search/refuel-planner.tsx @@ -107,11 +107,23 @@ export function RefuelPlanner({ const [timeDraft, setTimeDraft] = useState(null); // EV plans use money only when some charger the filters leave has a price in - // the display currency. Chargers without a price still compete, at 0: most + // the display currency. Chargers without a price still compete: most // countries publish none, and dropping them could leave a cross-border route - // with no plan. With no such priced charger, only detour and charging time count. - const evPriced = ev && stations.some(({ properties: p }) => - evEligible(p, maxDetour, minChargerKw) && p.price != null && p.price > 0 && p.currency === currency && (maxPrice == null || p.price <= maxPrice)); + // with no plan. They are costed at the median of the priced ones, so they + // are neither free (and picked everywhere) nor penalised. With no such + // priced charger, only detour and charging time count. + const evMedianPrice = useMemo(() => { + if (!ev) return null; + const prices = stations + .map(({ properties: p }) => p) + .filter((p) => evEligible(p, maxDetour, minChargerKw) && p.price != null && p.price > 0 && p.currency === currency && (maxPrice == null || p.price <= maxPrice)) + .map((p) => p.price as number) + .sort((a, b) => a - b); + if (prices.length === 0) return null; + const mid = Math.floor(prices.length / 2); + return prices.length % 2 ? prices[mid] : (prices[mid - 1] + prices[mid]) / 2; + }, [ev, stations, maxDetour, minChargerKw, currency, maxPrice]); + const evPriced = evMedianPrice != null; const usesMoney = !ev || evPriced; // The time value is stored in EUR: without a rate for the display currency it @@ -141,7 +153,8 @@ export function RefuelPlanner({ if (p.detourMin == null || !evEligible(p, maxDetour, minChargerKw)) return []; if (maxPrice != null && p.price != null && p.price > maxPrice && p.currency === currency) return []; const base = { id: p.id, km: (p.routeFraction ?? 0) * routeKm, detourMin: p.detourMin, minutesPerPct: minutesPerPct(evProfile, p.powerKw) }; - if (!evPriced || p.price == null || p.price <= 0) return [{ ...base, price: 0 }]; + if (!evPriced) return [{ ...base, price: 0 }]; + if (p.price == null || p.price <= 0) return [{ ...base, price: evMedianPrice }]; if (p.currency !== currency) return []; // The session fee is in the tariff's own currency; the price is already converted. const fee = p.evFees?.sessionFee; @@ -165,7 +178,7 @@ export function RefuelPlanner({ maxStops: maxStopsFor(routeKm, capacity, consumption, reservePct, maxChargePct), maxChargePct, }); - }, [open, ratesMissing, detoursPending, routeKm, stations, ev, evPriced, usesMoney, evProfile, minChargerKw, maxPrice, maxDetour, currency, convert, capacity, consumption, maxChargePct, startPct, arrival, reservePct, timeValue]); + }, [open, ratesMissing, detoursPending, routeKm, stations, ev, evPriced, evMedianPrice, usesMoney, evProfile, minChargerKw, maxPrice, maxDetour, currency, convert, capacity, consumption, maxChargePct, startPct, arrival, reservePct, timeValue]); useEffect(() => { const stops = plan?.status === "ok" @@ -388,7 +401,10 @@ export function RefuelPlanner({ {ev ? `${kwh(plan.stops.reduce((sum, s) => sum + s.litres, 0))} · ${minutes(plan.totalChargeMin)}` + - (evPriced ? ` · ${money(plan.totalFuelCost)}` : "") + // An unpriced stop is costed at an estimate, so no total is shown then. + (evPriced && plan.stops.every((s) => (byId.get(s.id)?.properties.price ?? 0) > 0) + ? ` · ${money(plan.totalFuelCost)}` + : "") : money(plan.totalFuelCost)} From 83271679b0ffe5b6dd235f3f667e30b8063bafc6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:17:10 +0200 Subject: [PATCH 22/25] fix(ocpdb): skip the odd bad record, cap per-minute fees at 0.30, anchor pins by position - One unparseable record failed every run, so a single bad location from a feed would freeze Germany's prices and sweep until a release. Bad records are skipped and counted again; the run fails only when more than 0.5 % of a collection does not parse, which means the feed's shape changed. - The 1.00/min cap let fees like 0.40/min from minute 0 and 0.60/min after 5 min onto the map. Real blocking fees run 0.05-0.24, so the cap is 0.30. - Pins anchored on upstream order, which the API does not promise, so two nearby feed sites could swap anchors and re-mint the pin's id (breaking shared links). Equal ranks now go by coordinates. --- docs/data/ev-sources.md | 6 ++++- src/scrapers/ocpdb.test.ts | 42 +++++++++++++++++++++++++++++---- src/scrapers/ocpdb.ts | 48 +++++++++++++++++++++++++++----------- 3 files changed, 76 insertions(+), 20 deletions(-) diff --git a/docs/data/ev-sources.md b/docs/data/ev-sources.md index eb349f7..581580d 100644 --- a/docs/data/ev-sources.md +++ b/docs/data/ev-sources.md @@ -243,7 +243,11 @@ A charger can have several tariffs. The pin shows one: The fees on top of the per-kWh price are stored in [`ev_fees`](../reference/data-model.md#ev_fees): - **Session fee:** fixed fees that always apply are added up. When the fee depends on the time of day, the weekday or other conditions, every option is kept and the popup lists them. The highest is the one used to rank tariffs and to cost a planned stop. -- **Per-minute fee:** the earliest-starting time-based charge that costs anything, with the minute it starts. These are blocking fees, typically 30 minutes to 4 hours into the session. OCPI defines them per hour, but every feed seen uses them per minute, so Pumperly reads them per minute. +- **Per-minute fee:** the earliest-starting time-based charge that costs anything, with the minute it starts. These are blocking fees, typically 30 minutes to 4 hours into the session. OCPI defines them per hour, but every feed seen uses them per minute, so Pumperly reads them per minute. Two feed errors are dropped: a time charge equal to the tariff's kWh price, and anything above 0.30 per minute, which no operator charges. + +### Records that do not parse {#ocpdb-bad-records} + +A record the scraper cannot read is skipped and counted in the run's log line, so one bad location from a feed does not stop Germany from updating. When more than 0.5 % of the tariffs, the tariff associations or the locations fail to parse, the feed's shape has most likely changed: the run fails and writes nothing, and the stations and prices from the last good run stay. ### When the feed health is unavailable {#ocpdb-feed-health} diff --git a/src/scrapers/ocpdb.test.ts b/src/scrapers/ocpdb.test.ts index 52e07b7..54cfee9 100644 --- a/src/scrapers/ocpdb.test.ts +++ b/src/scrapers/ocpdb.test.ts @@ -185,15 +185,26 @@ describe("summarizeTariff", () => { expect(summarizeTariff(tariff)?.fees).toEqual({ perMinute: 0.05, afterMin: 240 }); }); - // EcoMovement 133247. - it("ignores an implausible per-minute fee", () => { + // EcoMovement 133247 (14.28/min) and 133278 (0.397/min from minute 0). + it.each([14.28, 0.397])("ignores an implausible per-minute fee of %s", (fee) => { const tariff: Tariff = { ...CHARGECLOUD, - elements: [{ price_components: [{ type: "ENERGY", price: 0.59 }, { type: "TIME", price: 14.28 }] }], + elements: [{ price_components: [{ type: "ENERGY", price: 0.59 }, { type: "TIME", price: fee }] }], }; expect(summarizeTariff(tariff)?.fees).toEqual({}); }); + it("keeps a blocking fee at the cap", () => { + const tariff: Tariff = { + ...CHARGECLOUD, + elements: [ + { price_components: [{ type: "ENERGY", price: 0.59 }] }, + { price_components: [{ type: "TIME", price: 0.3 }], restrictions: { min_duration: 3600 } }, + ], + }; + expect(summarizeTariff(tariff)?.fees).toEqual({ perMinute: 0.3, afterMin: 60 }); + }); + // ChargePoint 132894. it("is null when unrestricted energy prices disagree", () => { const tariff: Tariff = { @@ -324,6 +335,16 @@ describe("liveSources", () => { }); describe("mergeLocations", () => { + // Two feed sites 10 m apart with no register site: whichever comes first + // anchors the pin. The API promises no order, so the anchor must not depend on it. + it("anchors the same pin whatever order the feed lists its sites in", () => { + const a = location({ source: "datex2_ecomovement", lat: 48.7758, evses: ["a"] }); + const b = location({ source: "datex2_ecomovement", lat: 48.7758 + 10 * METRES, evses: ["b"] }); + const ids = (locs: Location[]) => mergeLocations(locs, new Map()).stations.map((st) => st.externalId); + expect(ids([a, b])).toEqual(ids([b, a])); + expect(ids([a, b])).toHaveLength(1); + }); + const tariffs = evseTariffs( [ENBW, CHARGECLOUD], [ @@ -525,6 +546,8 @@ describe("OcpdbScraper.fetch", () => { failOnce?: string; failAlways?: string; broken?: boolean; + /** Extra valid locations on the first page, ~111 m apart so none merge. */ + extra?: number; } = {}, ) { const urls: string[] = []; @@ -561,6 +584,7 @@ describe("OcpdbScraper.fetch", () => { // With `broken`, the first page also carries one record the schema rejects. const items: unknown[] = [location({ lat: 48 + offset / 1000, evses: [offset === 0 ? "enbw" : "cc"] })]; if (offset === 0 && opts.broken) items.push({ source: "broken" }); + if (offset === 0) for (let i = 0; i < (opts.extra ?? 0); i++) items.push(location({ lat: 50 + i / 1000, evses: ["cc"] })); return page(items, opts.locationsTotal ?? 1); }), ); @@ -579,11 +603,19 @@ describe("OcpdbScraper.fetch", () => { expect(stations).toHaveLength(3); }); - it("fails the run on a record that does not parse, so the current prices stay", async () => { + it("skips a lone record that does not parse, so one bad location cannot freeze the country", async () => { + stubApi({ broken: true, extra: 300 }); + const { OcpdbScraper } = await import("./ocpdb"); + + const { stations } = await new OcpdbScraper().fetch(); + expect(stations).toHaveLength(301); + }); + + it("fails the run when many records do not parse, so the current prices stay", async () => { stubApi({ broken: true }); const { OcpdbScraper } = await import("./ocpdb"); - await expect(new OcpdbScraper().fetch()).rejects.toThrow(/1 unparseable OCPDB records/); + await expect(new OcpdbScraper().fetch()).rejects.toThrow(/1 of 2 locations unparseable/); }); // TotalEnergies tariffs on EcoMovement (132622, 132721, 132724, 133685) give diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts index e802916..f3c5b07 100644 --- a/src/scrapers/ocpdb.ts +++ b/src/scrapers/ocpdb.ts @@ -100,9 +100,17 @@ const DEAD_EVSE_STATUSES = new Set(["REMOVED", "PLANNED"]); // German standard VAT, for prices marked net that don't say at what rate. const DE_VAT_PCT = 19; -// Highest believable per-minute fee. Blocking fees seen run 0.05-0.24/min, -// so this leaves wide headroom while catching unit errors like 14.28/min. -const MAX_PER_MINUTE = 1; +// Highest believable per-minute fee. Blocking fees seen run 0.05-0.24/min. +// Above this the value is a unit or feed error (0.40/min from minute 0, +// 0.60/min after 5 min, 14.28/min), and showing it would scare drivers off +// a charger for a fee nobody charges. +const MAX_PER_MINUTE = 0.3; + +// Share of a collection that may fail to parse before the run gives up. +// Below it the odd record is skipped (one bad location must not freeze the +// country); above it the schema or the feed changed, and the current rows +// stay until someone looks. +const MAX_INVALID_SHARE = 0.005; // Session size used to fold a session fee into a per-kWh figure when ranking // tariffs: a typical fast-charge top-up. @@ -426,11 +434,17 @@ export function mergeLocations( ): { stations: RawStation[]; prices: RawFuelPrice[]; stats: OcpdbParseStats } { const stats: OcpdbParseStats = { locations: locations.length, notGermany: 0, noLiveEvse: 0, merged: 0, priced: 0 }; - // Stable sort: equal ranks keep upstream order, so the anchors (and the - // externalIds) don't shuffle between runs. + // Equal ranks go by coordinates, not upstream order, which the API does not + // promise: the anchors (and so the externalIds) must not shuffle between runs. const ordered = locations .map((loc, i) => ({ loc, i, rank: SOURCE_RANK[loc.source] ?? DEFAULT_RANK })) - .sort((a, b) => a.rank - b.rank || a.i - b.i) + .sort( + (a, b) => + a.rank - b.rank || + a.loc.coordinates.latitude - b.loc.coordinates.latitude || + a.loc.coordinates.longitude - b.loc.coordinates.longitude || + a.i - b.i, + ) .map((x) => x.loc); const grid = new Map(); @@ -589,10 +603,10 @@ async function fetchPage(path: string, offset: number): Promise(path: string, schema: z.ZodType): Promise<{ items: T[]; invalid: number }> { +async function fetchAll(path: string, schema: z.ZodType): Promise<{ items: T[]; invalid: number; total: number }> { const items: T[] = []; let invalid = 0; // Parse each page as it lands and let the raw JSON go: holding all ~100 @@ -614,7 +628,7 @@ async function fetchAll(path: string, schema: z.ZodType): Promise<{ items: const batch = await Promise.all(offsets.slice(i, i + CONCURRENCY).map((o) => fetchPage(path, o))); for (const p of batch) keep(p.items); } - return { items, invalid }; + return { items, invalid, total: items.length + invalid }; } /** @@ -651,10 +665,16 @@ export class OcpdbScraper extends BaseScraper { fetchAll("tariff-associations", AssociationSchema), fetchAll("locations?country=DEU", LocationSchema), ]); - // A dropped tariff or association would silently unprice its stations, and - // a dropped location would be swept: keep the current rows instead. + // A few bad records are skipped: their stations go unpriced or drop out + // until the feed fixes them. Many mean the shape changed, and a run on a + // fraction of the data would unprice or sweep the rest: keep the current rows. + const collections = { tariffs, associations, locations }; + for (const [name, c] of Object.entries(collections)) { + if (c.invalid > c.total * MAX_INVALID_SHARE) { + throw new Error(`[${this.source}] DE: ${c.invalid} of ${c.total} ${name} unparseable`); + } + } const invalid = tariffs.invalid + associations.invalid + locations.invalid; - if (invalid > 0) throw new Error(`[${this.source}] DE: ${invalid} unparseable OCPDB records`); const live = liveSources(sources, new Date(), MAX_FEED_AGE_DAYS); const stale = [...new Set(tariffs.items.map((t) => t.source))].filter((s) => !live.has(s)); if (stale.length > 0) { @@ -669,7 +689,7 @@ export class OcpdbScraper extends BaseScraper { console.log( `[${this.source}] DE: ${tariffs.items.length} tariffs → ${tariffsByEvse.size} priced EVSEs; ` + `${stats.locations} locations → ${stations.length} stations, ${stats.priced} priced ` + - `(merged ${stats.merged}, skipped ${stats.noLiveEvse} without live EVSE, ${stats.notGermany} outside DE)`, + `(merged ${stats.merged}, skipped ${stats.noLiveEvse} without live EVSE, ${stats.notGermany} outside DE, ${invalid} unparseable)`, ); return { stations, prices }; From 25cc4c58eb494f84f30b8c6dfc2eba6c15e2fe31 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:33:32 +0200 Subject: [PATCH 23/25] fix(ocpdb): never show an AC rate on a DC pin, and follow OCPI's first match for time fees - When no EVSE at a pin's top power had a price, the cheapest EVSE of any power stood in, so 101 DC pins showed an AC rate next to "150 kW" (Mer, ENTEGA). The stand-in must now be of the same class (DC from 50 kW); a DC pin with only an AC price stays unpriced. - Chargecloud lists an always-on TIME 0.00, then the real fee after 4 h, then always-on copies of the fee. We took a copy as the fee and showed 0.06/min from minute 0 on 251 pins. OCPI takes the first always-on TIME element, so later always-on ones are now ignored. --- docs/data/ev-sources.md | 4 ++-- src/scrapers/ocpdb.test.ts | 34 ++++++++++++++++++++++++++++++++-- src/scrapers/ocpdb.ts | 28 +++++++++++++++++++++++++--- 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/docs/data/ev-sources.md b/docs/data/ev-sources.md index 581580d..9034fc3 100644 --- a/docs/data/ev-sources.md +++ b/docs/data/ev-sources.md @@ -238,12 +238,12 @@ A charger can have several tariffs. The pin shows one: 2. **Only tariffs from healthy feeds count.** A feed that OCPDB marks as failed, or that has not refreshed in [`PUMPERLY_OCPDB_MAX_FEED_AGE_DAYS`](../reference/environment-variables.md#pumperly_ocpdb_max_feed_age_days) days, has its tariffs ignored, and its chargers show unpriced. The price's `reported_at` is when its feed last refreshed. 3. **Prices are gross.** Ad-hoc charging is a consumer price, and German law requires it shown with VAT. A price marked net gets the VAT rate it lists, or 19 % when it lists none. A price with no tax data is read as gross. 4. **Prices outside EUR 0.10 to 2.00 per kWh are dropped** before the cheapest is picked, so a bogus 0.00 cannot hide a real price. -5. **The cheapest wins,** with the session fee spread over a 30 kWh session. A low rate cannot hide a large fee. +5. **The cheapest at the pin's top power wins,** with the session fee spread over a 30 kWh session, so a low rate cannot hide a large fee. The price shown is the one paid at the kW shown next to it. When no charger at the top power has a price, the cheapest of the same class stands in (DC from 50 kW, AC below). A DC pin never shows an AC rate: it stays unpriced instead. The fees on top of the per-kWh price are stored in [`ev_fees`](../reference/data-model.md#ev_fees): - **Session fee:** fixed fees that always apply are added up. When the fee depends on the time of day, the weekday or other conditions, every option is kept and the popup lists them. The highest is the one used to rank tariffs and to cost a planned stop. -- **Per-minute fee:** the earliest-starting time-based charge that costs anything, with the minute it starts. These are blocking fees, typically 30 minutes to 4 hours into the session. OCPI defines them per hour, but every feed seen uses them per minute, so Pumperly reads them per minute. Two feed errors are dropped: a time charge equal to the tariff's kWh price, and anything above 0.30 per minute, which no operator charges. +- **Per-minute fee:** the earliest-starting time-based charge that costs anything, with the minute it starts. These are blocking fees, typically 30 minutes to 4 hours into the session. OCPI defines them per hour, but every feed seen uses them per minute, so Pumperly reads them per minute. As in OCPI, the first time charge that always applies wins, so later copies of it are ignored (Chargecloud lists a free one first, then the real fee after 4 hours, then copies). Two feed errors are dropped: a time charge equal to the tariff's kWh price, and anything above 0.30 per minute, which no operator charges. ### Records that do not parse {#ocpdb-bad-records} diff --git a/src/scrapers/ocpdb.test.ts b/src/scrapers/ocpdb.test.ts index 54cfee9..381cadc 100644 --- a/src/scrapers/ocpdb.test.ts +++ b/src/scrapers/ocpdb.test.ts @@ -185,6 +185,28 @@ describe("summarizeTariff", () => { expect(summarizeTariff(tariff)?.fees).toEqual({ perMinute: 0.05, afterMin: 240 }); }); + // Chargecloud 133380: an always-on TIME 0.00 first, the real fee after 4 h, + // then always-on copies of it. By OCPI's first match the copies never apply. + it("ignores always-on TIME components after the first one", () => { + const time = (price: number, min_duration?: number) => ({ + price_components: [{ type: "TIME", price }], + ...(min_duration ? { restrictions: { min_duration } } : {}), + }); + const tariff: Tariff = { + ...CHARGECLOUD, + elements: [ + { price_components: [{ type: "FLAT", price: 0.85 }] }, + { price_components: [{ type: "ENERGY", price: 0.44 }] }, + time(0), + time(0.06, 14400), + time(0, 26400), + time(0), + time(0.06), + ], + }; + expect(summarizeTariff(tariff)?.fees).toEqual({ sessionFee: 0.85, perMinute: 0.06, afterMin: 240 }); + }); + // EcoMovement 133247 (14.28/min) and 133278 (0.397/min from minute 0). it.each([14.28, 0.397])("ignores an implausible per-minute fee of %s", (fee) => { const tariff: Tariff = { @@ -415,10 +437,18 @@ describe("mergeLocations", () => { expect(prices[0]).toMatchObject({ price: 0.79, evFees: { perMinute: 0.24, afterMin: 30 } }); }); - it("falls back to the cheapest EVSE when none at the top power is priced", () => { + it("falls back to a cheaper EVSE of the same class when none at the top power is priced", () => { + const { stations, prices } = mergeLocations([site({ unpriced: 150, dc: 50 })], hpc); + expect(stations[0].maxPowerKw).toBe(150); + expect(prices[0].price).toBe(0.79); + }); + + // Like ocpdb-52.65288_9.21849 (Mer): 150 kW unpriced, 22 kW priced. An AC + // rate next to "150 kW" would undersell the fast charge. + it("leaves a DC pin unpriced rather than show an AC rate", () => { const { stations, prices } = mergeLocations([site({ unpriced: 150, ac: 22 })], hpc); expect(stations[0].maxPowerKw).toBe(150); - expect(prices[0].price).toBe(0.7); + expect(prices).toHaveLength(0); }); it("applies the same rule across merged locations", () => { diff --git a/src/scrapers/ocpdb.ts b/src/scrapers/ocpdb.ts index f3c5b07..69834c1 100644 --- a/src/scrapers/ocpdb.ts +++ b/src/scrapers/ocpdb.ts @@ -259,6 +259,11 @@ export function summarizeTariff(tariff: OcpdbTariff): TariffSummary | null { let sessionFee = 0; const rules: SessionFeeRule[] = []; let perMinute: { price: number; afterSec: number } | null = null; + // OCPI takes the first element that matches, per dimension: after one + // always-on TIME component (even a free one), later always-on ones can never + // apply. Chargecloud lists TIME 0.00, then the real fee after 4 h, then + // always-on copies of the fee or the kWh rate (133380, 133462). + let unrestrictedTimeSeen = false; const unrestrictedEnergy = new Set(); const energyPrices = new Set( tariff.elements.flatMap((e) => e.price_components.filter((c) => c.type === "ENERGY").map((c) => grossPrice(c, tariff))), @@ -277,7 +282,12 @@ export function summarizeTariff(tariff: OcpdbTariff): TariffSummary | null { } else if (c.type === "FLAT") { if (!restricted) sessionFee += price; else rules.push(feeRule(price, element.restrictions ?? {})); - } else if (c.type === "TIME" && price > 0 && price <= MAX_PER_MINUTE && !energyPrices.has(price)) { + } else if (c.type === "TIME") { + if (!restricted) { + if (unrestrictedTimeSeen) continue; + unrestrictedTimeSeen = true; + } + if (price <= 0 || price > MAX_PER_MINUTE || energyPrices.has(price)) continue; // Several TIME steps: the first one that costs anything is the fee // a driver will actually meet. if (perMinute == null || afterSec < perMinute.afterSec) perMinute = { price, afterSec }; @@ -396,6 +406,16 @@ export interface OcpdbParseStats { interface Offer { tariff: TariffSummary; operator: string | null; + /** Power of the EVSE the tariff was found on. */ + powerKw: number | null; +} + +// From here a charger counts as fast (DC). A fallback price must come from an +// EVSE of the same class as the pin's power, or a 150 kW pin shows a 22 kW rate. +const FAST_KW = 50; + +function sameClass(a: number | null | undefined, b: number | null | undefined): boolean { + return (a ?? 0) >= FAST_KW === (b ?? 0) >= FAST_KW; } interface Pin { @@ -482,7 +502,7 @@ export function mergeLocations( for (let i = 0; i < evses.length; i++) { const t = tariffsByEvse.get(evses[i].uid); if (!t) continue; - const offer = { tariff: t, operator }; + const offer = { tariff: t, operator, powerKw: evsePower[i] }; any = cheaperOffer(offer, any); if (powerKw != null && evsePower[i] === powerKw) top = cheaperOffer(offer, top); } @@ -551,7 +571,9 @@ export function mergeLocations( const prices: RawFuelPrice[] = []; for (const { station, top, any, site } of pins) { - const offer = top ?? any; + // No priced EVSE at the top power: a cheaper one of the same class may + // stand in, never an AC rate on a DC pin (or the other way round). + const offer = top ?? (any && sameClass(any.powerKw, station.maxPowerKw) ? any : null); if (!offer) continue; const { tariff, operator } = offer; if (operator && operator !== station.brand) { From 9d8a585bf91fe9830b2544b8ffcdcb0ffda7ce1b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:33:34 +0200 Subject: [PATCH 24/25] fix(planner): leave chargers with an unknown detour out of the median price A priced charger whose detour had not come back yet still switched the plan to money and set the stand-in price, though the plan itself could not use that charger. Only chargers the plan can use count now. --- src/components/search/refuel-planner.test.tsx | 12 ++++++++++++ src/components/search/refuel-planner.tsx | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/components/search/refuel-planner.test.tsx b/src/components/search/refuel-planner.test.tsx index 39d3f6d..f03ee95 100644 --- a/src/components/search/refuel-planner.test.tsx +++ b/src/components/search/refuel-planner.test.tsx @@ -458,6 +458,18 @@ describe("RefuelPlanner", () => { expect(screen.queryByText(/ · \d+\.\d\d €$/)).not.toBeInTheDocument(); }); + it("plans by time when the only priced charger has no detour yet", async () => { + const chargers = [ + makeStation("pending", { price: 0.59, fuelType: "EV", routeFraction: 0.3, detourMin: undefined }), + makeStation("free", { price: null, currency: undefined, fuelType: "EV", routeFraction: 0.6, detourMin: 0 }), + ]; + renderPlanner({ mode: "ev", stations: chargers, routeKm: 300 }); + await userEvent.click(screen.getByText("planner.titleEv")); + const input = planSpy.mock.lastCall![0]; + expect(input.stations).toEqual([expect.objectContaining({ id: "free", price: 0 })]); + expect(input.timeValuePerHour).toBe(1); + }); + it("still drops chargers priced in another currency or above the price filter", async () => { const chargers = [ makeStation("ok", { price: 0.59, fuelType: "EV", routeFraction: 0.3, detourMin: 0 }), diff --git a/src/components/search/refuel-planner.tsx b/src/components/search/refuel-planner.tsx index 9dcda4b..b3c5da1 100644 --- a/src/components/search/refuel-planner.tsx +++ b/src/components/search/refuel-planner.tsx @@ -116,7 +116,8 @@ export function RefuelPlanner({ if (!ev) return null; const prices = stations .map(({ properties: p }) => p) - .filter((p) => evEligible(p, maxDetour, minChargerKw) && p.price != null && p.price > 0 && p.currency === currency && (maxPrice == null || p.price <= maxPrice)) + // Only chargers the plan can use: a detour still unknown keeps one out. + .filter((p) => p.detourMin != null && evEligible(p, maxDetour, minChargerKw) && p.price != null && p.price > 0 && p.currency === currency && (maxPrice == null || p.price <= maxPrice)) .map((p) => p.price as number) .sort((a, b) => a - b); if (prices.length === 0) return null; From 16566c726ed5dfca89ef21bf77d9e8d665775713 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:33:35 +0200 Subject: [PATCH 25/25] docs: give the app 1.5 GB in the sample compose The scrapers run inside the app and Node caps its heap at about half the container limit. With 512M the heap runs out while Germany's EV source runs next to other scrapers, which is how the app gets killed. --- docs/getting-started/full-stack.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/getting-started/full-stack.md b/docs/getting-started/full-stack.md index a6eedbf..81a841e 100644 --- a/docs/getting-started/full-stack.md +++ b/docs/getting-started/full-stack.md @@ -277,7 +277,10 @@ services: deploy: resources: limits: - memory: 512M + # The scrapers run inside the app, and Node caps its heap at about + # half of this. Germany's EV sources (BNetzA, OCPDB) need ~170 MB of + # heap on their own, so 512M runs out while other scrapers are busy. + memory: 1536M volumes: pumperly-pgdata: