diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9677eca..35a6d0b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,9 +5,9 @@ name: Release # The tag is pushed by a human after the PR is merged: publishing a version # engages the public contract (format + schema), so it is never automated. # The release carries, under ODbL 1.0 (two IDFM sources are ODbL): -# - stations.format-1.json export of ALL modes (the public offer of -# the derived database) -# - stations.format-1.schema.json the closed JSON Schema of format 1 +# - stations.format-.json export of ALL modes, for every produced +# format (the public offer of the derived database) +# - stations.format-.schema.json the closed JSON Schema of each format # Consumers that embed a subset re-run the exporter with --modes; `generator` # and `modes` in the file are enough to reproduce it. @@ -45,18 +45,22 @@ jobs: uv run ruff format --check . uv run pytest - - name: Export the referential (all modes) and copy its schema + - name: Export the referential (all modes, every format) and copy the schemas run: | mkdir -p dist - uv run export-referential --format 1 --output dist/stations.format-1.json - cp prim_api/referential/schemas/stations.format-1.schema.json dist/ + # The latest format downloads all its datasets (a superset of the older ones); the + # other formats reuse them (--no-download), so every file describes the same data. + uv run export-referential --format 2 --output dist/stations.format-2.json + uv run export-referential --no-download --format 1 --output dist/stations.format-1.json + cp prim_api/referential/schemas/stations.format-*.schema.json dist/ - name: Create the GitHub release env: GH_TOKEN: ${{ github.token }} run: | cat > notes.md <<'NOTES' - Stations referential of Île-de-France (all modes) and its JSON Schema. + Stations referential of Île-de-France (all modes), in every produced format, + with the JSON Schema of each format. **Licence**: the referential is a derived database of Île-de-France Mobilités open data and is published under the [ODbL 1.0](https://opendatacommons.org/licenses/odbl/1-0/). diff --git a/CLAUDE.md b/CLAUDE.md index d143a7b..62bc17a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,11 +58,11 @@ tools/ # CLI scripts: sync_specs, generate_clients, sync_datasets, - **Pipeline order** (`sync_all.py`): sync_specs → generate_clients → sync_datasets → validate_datasets. - **Python SDK** (`prim_api/`): `IdFMPrimAPI` wraps the generated client with a clean interface, auto-downloads datasets on first use, and refreshes them in a background thread. Core dataset logic lives in `prim_api/datasets.py` (shared with CLI tools). - **Refs module** (`prim_api/refs.py`): IDFM ↔ STIF identifier conversion helpers (`StopPointRef`, `StopAreaRef`, `LineRef`, `parse_stop_ref`, `parse_line_ref`). -- **Referential exporter** (`prim_api/referential/`): builds the stations referential from five datasets (`zones-de-correspondance`, `zones-d-arrets`, `arrets`, `arrets-lignes`, `referentiel-des-lignes`). `format1.py` is pure (no network, no files) and holds every construction rule; `cli.py` (`export-referential`) downloads, reads `data_processed` from the portal metadata, builds, validates against the schema, writes. `import prim_api` must stay light: `IdFMPrimAPI` is imported lazily because the generated client only exists in a checkout. +- **Referential exporter** (`prim_api/referential/`): format 1 from five datasets, format 2 (format 1 + accesses, `format2.py`) from seven; the CLI keeps a `FORMATS` registry (sources, builder). It builds the stations referential from five datasets (`zones-de-correspondance`, `zones-d-arrets`, `arrets`, `arrets-lignes`, `referentiel-des-lignes`). `format1.py` is pure (no network, no files) and holds every construction rule; `cli.py` (`export-referential`) downloads, reads `data_processed` from the portal metadata, builds, validates against the schema, writes. `import prim_api` must stay light: `IdFMPrimAPI` is imported lazily because the generated client only exists in a checkout. ## Referential contract rules (do not break) -- The format is **strict**: any change of shape (field, mode value, construction rule, sort order) is a **new format number** with its own schema file; never edit `stations.format-1.schema.json` or the format 1 rules once `v1.0.0` is tagged. +- The format is **strict**: any change of shape (field, mode value, construction rule, sort order) is a **new format number** with its own schema file; never edit a released format's schema or rules (format 1 frozen by `v1.0.0`, format 2 by `v1.1.0`). A new format keeps producing the previous ones (minor release). - The exporter stays **generic**: options are `--format` and `--modes` only; no city, profile or consumer-specific notion. - Source inconsistencies raise `ReferentialError`; never drop data silently. - SemVer: new format while keeping the old ones = minor; dropping a format = major. Update the README "Changes" section in every PR that changes behaviour. diff --git a/README.md b/README.md index 8a92540..07d8f20 100644 --- a/README.md +++ b/README.md @@ -93,18 +93,18 @@ about cities or consumers: it always covers the whole network, and its only opti ```bash # From a tag, without cloning (what consumers should pin): -uvx --from git+https://github.com/MaxLeb/idfm-prim-api@v1.0.0 \ - export-referential --format 1 --modes METRO,RER,TRAIN,TRAM > stations.json +uvx --from git+https://github.com/MaxLeb/idfm-prim-api@v1.1.0 \ + export-referential --format 2 --modes METRO,RER,TRAIN,TRAM > stations.json -# From a checkout: -uv run export-referential --modes METRO,RER -o stations.json -uv run export-referential --no-download -o stations.json # reuse downloaded datasets +# From a checkout (always pass --format: the default stays 1 for compatibility): +uv run export-referential --format 2 --modes METRO,RER -o stations.json +uv run export-referential --format 2 --no-download -o stations.json # reuse downloaded datasets ``` Datasets are downloaded to `data/raw/` in a checkout, or to `~/.cache/idfm-prim-api/raw` (`$XDG_CACHE_HOME`) when installed. The file is validated against its schema before being written. -Each [release](https://github.com/MaxLeb/idfm-prim-api/releases) also publishes the export of all -modes and the schema. +Each [release](https://github.com/MaxLeb/idfm-prim-api/releases) also publishes, for every produced +format, the export of all modes and its schema. ### Format 1 @@ -159,16 +159,42 @@ Sources: `zones-de-correspondance`, `zones-d-arrets`, `arrets` (Licence Ouverte `arrets-lignes`, `referentiel-des-lignes` (ODbL). Any inconsistency in them (unresolvable stop, line, area or hub, unknown mode) fails the export rather than dropping data silently. +### Format 2 + +Format 2 is format 1 plus, for every station, the **accesses** (entrances and exits) of its stop +areas, from the IDFM datasets `acces` and `relations-acces` (Licence Ouverte 2.0). Consumers that do +not need them keep asking for `--format 1`, which stays produced. + +```json +{ "id": "IDFM:474151", "name": "Châtelet - Les Halles", "…": "…", + "accesses": [ + { "id": "IDFM:50148652", "name": "Porte Marguerite de Navarre", "number": 1, + "lat": 48.860433, "lon": 2.346346, "entry": true, "exit": true } + ] } +``` + +| Field | Meaning | +|---|---| +| `accesses[].id` | `IDFM:`: the id that platform positioning (`positionnement-dans-la-rame`) points to. | +| `name`, `number` | Name as published; number shown on the signage, or `null`. | +| `lat`, `lon` | Published WGS 84 point, six decimals. | +| `entry`, `exit` | Whether one can enter, exit, or both. | + +Rules: every access linked to one of the station's `areas` (the stop areas of the kept modes), each +listed once, sorted by numeric id; `[]` when none is published (tram stops, a few TER stations +outside Île-de-France). The `version` is the most recent `data_processed` of the **seven** datasets. + ### Versioning - **The format is strict**: every field is always present (`null` when the source has no value), - the schema is closed (`prim_api/referential/schemas/stations.format-1.schema.json`), and **any** + each schema is closed (`prim_api/referential/schemas/stations.format-.schema.json`), and **any** change of shape — a field, a mode value, a construction rule — is a **new format number**. Consumers read exactly the format they know and refuse the others. - **The repository follows SemVer on that contract**: producing a new format while still producing the previous ones is a **minor** release; dropping a format is a **major** release; a fix that keeps format and output unchanged is a **patch**. -- Format 1 is frozen by `v1.0.0`. Tags are pushed by a maintainer, never by automation. +- Format 1 is frozen by `v1.0.0`, format 2 by `v1.1.0`. Tags are pushed by a maintainer, never by + automation. - The format is owned by its first consumer (the Everyday France app); this repository is its first producer and hosts its schema. @@ -181,6 +207,15 @@ Releases make the full export available; a filtered export is reproducible from ## Changes +### v1.1.0 + +- New: **format 2** (`--format 2`, `prim_api.referential.format2`, closed schema + `stations.format-2.schema.json`): format 1 plus the accesses of each station. Format 1 is still + produced unchanged. +- New datasets in `manifests/datasets.yml`: `acces`, `relations-acces`. +- The release publishes every format (all modes) and every schema. +- CLI: one registry of formats (sources and builder); the summary counts accesses for format 2. + ### v1.0.0 - New: `export-referential` and `prim_api.referential` — stations referential, **format 1**, closed diff --git a/data/schema/acces.schema.json b/data/schema/acces.schema.json new file mode 100644 index 0000000..4b798cf --- /dev/null +++ b/data/schema/acces.schema.json @@ -0,0 +1,98 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "accid": { + "type": [ + "string", + "null" + ] + }, + "accversion": { + "type": [ + "string", + "null" + ] + }, + "acccreated": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "accchanged": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "fournisseurid": { + "type": [ + "string", + "null" + ] + }, + "fournisseurname": { + "type": [ + "string", + "null" + ] + }, + "accname": { + "type": [ + "string", + "null" + ] + }, + "accshortname": { + "type": [ + "integer", + "null" + ] + }, + "accdescription": { + "type": [ + "string", + "null" + ] + }, + "accprivatecode": { + "type": [ + "string", + "null" + ] + }, + "accxespg2154": { + "type": [ + "integer", + "null" + ] + }, + "accyespg2154": { + "type": [ + "integer", + "null" + ] + }, + "accisentry": { + "type": [ + "string", + "null" + ] + }, + "accisexit": { + "type": [ + "string", + "null" + ] + }, + "accgeopoint": { + "type": [ + "object", + "null" + ] + } + } +} \ No newline at end of file diff --git a/data/schema/relations-acces.schema.json b/data/schema/relations-acces.schema.json new file mode 100644 index 0000000..f597437 --- /dev/null +++ b/data/schema/relations-acces.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "zdaid": { + "type": [ + "string", + "null" + ] + }, + "version": { + "type": [ + "string", + "null" + ] + }, + "accid": { + "type": [ + "string", + "null" + ] + }, + "accversion": { + "type": [ + "string", + "null" + ] + } + } +} \ No newline at end of file diff --git a/manifests/datasets.yml b/manifests/datasets.yml index 4be3fa9..68ab508 100644 --- a/manifests/datasets.yml +++ b/manifests/datasets.yml @@ -21,3 +21,12 @@ datasets: portal_base: "https://data.iledefrance-mobilites.fr" export_format: "jsonl" validate: true + # Format 2 of the referential: entrances and exits of stations, and their stop areas. + - dataset_id: "acces" + portal_base: "https://data.iledefrance-mobilites.fr" + export_format: "jsonl" + validate: true + - dataset_id: "relations-acces" + portal_base: "https://data.iledefrance-mobilites.fr" + export_format: "jsonl" + validate: true diff --git a/prim_api/referential/__init__.py b/prim_api/referential/__init__.py index b7b61fc..5ce641e 100644 --- a/prim_api/referential/__init__.py +++ b/prim_api/referential/__init__.py @@ -5,6 +5,7 @@ numbered, strict format validated by a closed JSON Schema. - ``format1``: construction rules of format 1 (pure, unit-tested); +- ``format2``: format 1 plus the accesses (entrances, exits) of each station; - ``schema``: the JSON Schemas shipped with the package; - ``cli``: the ``export-referential`` command. @@ -25,6 +26,7 @@ map_mode, parse_modes, ) +from prim_api.referential.format2 import build_stations_file_format2 from prim_api.referential.schema import load_schema, validate_stations_file __all__ = [ @@ -36,6 +38,7 @@ "SOURCE_DATASETS", "ReferentialError", "build_stations_file", + "build_stations_file_format2", "load_schema", "map_mode", "parse_modes", diff --git a/prim_api/referential/cli.py b/prim_api/referential/cli.py index 9249ab4..ed46330 100644 --- a/prim_api/referential/cli.py +++ b/prim_api/referential/cli.py @@ -34,13 +34,12 @@ from rich.console import Console from prim_api import datasets +from prim_api.referential import format1, format2 from prim_api.referential.format1 import ( FORMAT, MODES, PORTAL_BASE, - SOURCE_DATASETS, ReferentialError, - build_stations_file, parse_modes, ) from prim_api.referential.schema import SCHEMA_FILES, validate_stations_file @@ -51,6 +50,13 @@ #: Distribution name in ``pyproject.toml`` (used to read the installed version). DISTRIBUTION_NAME = "prim-api" +#: Each produced format: its source datasets and its builder. Adding a format while keeping the +#: others is a minor release; dropping one is a major release (README « Versioning »). +FORMATS = { + 1: (format1.SOURCE_DATASETS, format1.build_stations_file), + 2: (format2.SOURCE_DATASETS, format2.build_stations_file_format2), +} + app = typer.Typer(add_completion=False, help=__doc__.split("\n\n")[0]) # Everything but the document goes to stderr, so that stdout can be redirected. console = Console(stderr=True) @@ -78,10 +84,10 @@ def default_data_dir() -> Path: return cache_root / GENERATOR_NAME / "raw" -def _download(data_dir: Path) -> None: +def _download(data_dir: Path, sources: tuple[str, ...]) -> None: """Download every source dataset and record its ``data_processed`` date.""" with httpx.Client(timeout=60.0, follow_redirects=True) as client: - for dataset_id in SOURCE_DATASETS: + for dataset_id in sources: console.print(f"[dim]Downloading {dataset_id}…[/dim]") if not datasets.ensure_dataset(dataset_id, PORTAL_BASE, data_dir=data_dir): raise ReferentialError(f"download of {dataset_id} failed") @@ -93,10 +99,10 @@ def _download(data_dir: Path) -> None: datasets.record_metadata_fields(dataset_id, {"data_processed": processed}, data_dir) -def _read_data_processed(data_dir: Path) -> dict[str, str]: +def _read_data_processed(data_dir: Path, sources: tuple[str, ...]) -> dict[str, str]: """Read the ``data_processed`` dates recorded next to the datasets.""" dates: dict[str, str] = {} - for dataset_id in SOURCE_DATASETS: + for dataset_id in sources: processed = (datasets.load_metadata(dataset_id, data_dir) or {}).get("data_processed") if not processed: raise ReferentialError( @@ -106,10 +112,10 @@ def _read_data_processed(data_dir: Path) -> dict[str, str]: return dates -def _load_records(data_dir: Path) -> dict[str, list[dict]]: - """Load the five datasets from ``data_dir``; an empty dataset is an error.""" +def _load_records(data_dir: Path, sources: tuple[str, ...]) -> dict[str, list[dict]]: + """Load the source datasets from ``data_dir``; an empty dataset is an error.""" records = {} - for dataset_id in SOURCE_DATASETS: + for dataset_id in sources: rows = datasets.load_dataset(dataset_id, data_dir=data_dir) if not rows: raise ReferentialError(f"{dataset_id}: no records in {data_dir}") @@ -124,13 +130,20 @@ def _summary(document: dict) -> str: for mode in {line["mode"] for line in stop["lines"]}: counts[mode] += 1 per_mode = ", ".join(f"{mode} {count}" for mode, count in counts.items()) - return f"{len(document['stops'])} stations ({per_mode}), version {document['version']}" + summary = f"{len(document['stops'])} stations ({per_mode}), version {document['version']}" + if document["format"] >= 2: + accesses = sum(len(stop["accesses"]) for stop in document["stops"]) + summary += f", {accesses} accesses" + return f"format {document['format']}: {summary}" @app.command() def main( format_number: Annotated[ - int, typer.Option("--format", help=f"Format to produce (known: {sorted(SCHEMA_FILES)}).") + int, + typer.Option( + "--format", help=f"Format to produce ({sorted(SCHEMA_FILES)}); always pass it." + ), ] = FORMAT, modes: Annotated[ str, typer.Option("--modes", help="Comma-separated modes to keep (default: all).") @@ -151,19 +164,20 @@ def main( ) -> None: """Export the stations referential of Île-de-France.""" try: - if format_number not in SCHEMA_FILES: - known = sorted(SCHEMA_FILES) + if format_number not in FORMATS or format_number not in SCHEMA_FILES: + known = sorted(FORMATS) raise ReferentialError( f"format {format_number} is not produced by this version (known: {known})" ) + sources, build = FORMATS[format_number] selected = parse_modes(modes) directory = data_dir or default_data_dir() if download: - _download(directory) - document = build_stations_file( - _load_records(directory), + _download(directory, sources) + document = build( + _load_records(directory, sources), modes=selected, - data_processed=_read_data_processed(directory), + data_processed=_read_data_processed(directory, sources), generator=generator_string(), generated_at=datetime.now(UTC), ) diff --git a/prim_api/referential/format2.py b/prim_api/referential/format2.py new file mode 100644 index 0000000..88b40bc --- /dev/null +++ b/prim_api/referential/format2.py @@ -0,0 +1,145 @@ +"""Format 2 of the stations referential: format 1 plus the **accesses** of each station. + +Format 2 is format 1 (same stations, same construction rules, same order, see +``format1``) where every station also carries ``accesses``: the entrances and +exits of its stop areas, as published by IDFM in the ``acces`` dataset and +linked to stop areas by ``relations-acces``. Consumers use the entrances to +measure the walk to the nearest one; exits are kept too, because platform +positioning (``positionnement-dans-la-rame``) points to access ids. + +Rules that are part of format 2 (changing one is a new format number): + +- **Accesses of a station** = every access linked to one of its ``areas`` (the + stop areas of the selected modes), each listed once even when linked to two + areas, sorted by id. ``[]`` when none (tram stops, a few TER stations). +- **Access fields**: ``id`` (``IDFM:``), ``name`` (as published), + ``number`` (the number shown on the signage, ``None`` if unpublished), ``lat`` + and ``lon`` (the published WGS 84 point, six decimals), ``entry`` and + ``exit`` (booleans). +- **Version** = most recent ``data_processed`` among the **seven** source + datasets (the five of format 1, plus ``acces`` and ``relations-acces``). + +Any inconsistency (a relation to an unknown access, an access without a name, +coordinates or entry/exit flags) raises ``ReferentialError``. +""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Iterable, Mapping +from datetime import datetime +from typing import Any + +from prim_api.referential.format1 import ( + SOURCE_DATASETS as FORMAT1_SOURCE_DATASETS, +) +from prim_api.referential.format1 import ( + CoordinateConverter, + ReferentialError, + build_stations_file, + format_timestamp, + parse_timestamp, +) + +#: Format number written in the ``format`` header field. +FORMAT = 2 + +ACCES = "acces" +RELATIONS_ACCES = "relations-acces" + +#: The seven datasets format 2 is built from. +SOURCE_DATASETS: tuple[str, ...] = (*FORMAT1_SOURCE_DATASETS, ACCES, RELATIONS_ACCES) + +_BOOLEANS = {"true": True, "false": False} +_STOP_AREA_PREFIX = "IDFM:monomodalStopPlace:" + + +def _flag(access: Mapping[str, Any], key: str) -> bool: + """``"true"`` / ``"false"`` (as published) → bool; anything else is an error.""" + value = access.get(key) + if isinstance(value, bool): + return value + if value not in _BOOLEANS: + raise ReferentialError(f"acces: access {access.get('accid')!r} has invalid {key} {value!r}") + return _BOOLEANS[value] + + +def _access_entry(access: Mapping[str, Any]) -> dict[str, Any]: + """One access of format 2, validated.""" + access_id = access.get("accid") + name = (access.get("accname") or "").strip() + point = access.get("accgeopoint") or {} + if not access_id or not name: + raise ReferentialError(f"acces: access {access_id!r} lacks an id or a name") + if point.get("lat") is None or point.get("lon") is None: + raise ReferentialError(f"acces: access {access_id!r} lacks coordinates") + number = access.get("accshortname") + if number is not None and not isinstance(number, int): + raise ReferentialError(f"acces: access {access_id!r} has a non-integer number {number!r}") + return { + "id": f"IDFM:{access_id}", + "name": name, + "number": number, + "lat": round(float(point["lat"]), 6), + "lon": round(float(point["lon"]), 6), + "entry": _flag(access, "accisentry"), + "exit": _flag(access, "accisexit"), + } + + +def build_stations_file_format2( + records: Mapping[str, Iterable[Mapping[str, Any]]], + *, + modes: Iterable[str], + data_processed: Mapping[str, str], + generator: str, + generated_at: datetime, + to_wgs84: CoordinateConverter | None = None, +) -> dict[str, Any]: + """Build the stations referential, format 2 (format 1 + ``accesses``). + + Same arguments as ``format1.build_stations_file``, with the two access + datasets in ``records`` and ``data_processed``. + + Raises: + ReferentialError: On a missing dataset or any source inconsistency. + """ + missing = [d for d in SOURCE_DATASETS if d not in records or d not in data_processed] + if missing: + raise ReferentialError(f"missing source dataset(s) or data_processed: {missing}") + + document = build_stations_file( + records, + modes=modes, + data_processed=data_processed, + generator=generator, + generated_at=generated_at, + to_wgs84=to_wgs84, + ) + + accesses: dict[str, Mapping[str, Any]] = {} + for access in records[ACCES]: + access_id = str(access.get("accid") or "") + if not access_id or access_id in accesses: + raise ReferentialError(f"acces: missing or duplicate access id {access_id!r}") + accesses[access_id] = access + + by_area: dict[str, set[str]] = defaultdict(set) + for relation in records[RELATIONS_ACCES]: + access_id, area_id = str(relation.get("accid") or ""), str(relation.get("zdaid") or "") + if access_id not in accesses: + raise ReferentialError(f"relations-acces: access {access_id!r} not in acces") + by_area[area_id].add(access_id) + + for stop in document["stops"]: + ids: set[str] = set() + for area in stop["areas"]: + ids |= by_area.get(area.removeprefix(_STOP_AREA_PREFIX), set()) + # Sorted by id for stable diffs; numeric ids sort numerically. + ordered = sorted(ids, key=lambda value: (len(value), value)) + stop["accesses"] = [_access_entry(accesses[access_id]) for access_id in ordered] + + version = max(parse_timestamp(data_processed[d]) for d in SOURCE_DATASETS) + document["format"] = FORMAT + document["version"] = format_timestamp(version) + return document diff --git a/prim_api/referential/schema.py b/prim_api/referential/schema.py index 420d5b1..8fd3d80 100644 --- a/prim_api/referential/schema.py +++ b/prim_api/referential/schema.py @@ -14,7 +14,10 @@ import jsonschema #: Schema file of each supported format. -SCHEMA_FILES: dict[int, str] = {1: "stations.format-1.schema.json"} +SCHEMA_FILES: dict[int, str] = { + 1: "stations.format-1.schema.json", + 2: "stations.format-2.schema.json", +} def schema_filename(format_number: int) -> str: diff --git a/prim_api/referential/schemas/stations.format-2.schema.json b/prim_api/referential/schemas/stations.format-2.schema.json new file mode 100644 index 0000000..d822f28 --- /dev/null +++ b/prim_api/referential/schemas/stations.format-2.schema.json @@ -0,0 +1,231 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/MaxLeb/idfm-prim-api/prim_api/referential/schemas/stations.format-2.schema.json", + "title": "Stations referential, format 2", + "description": "Format 1 plus the accesses (entrances and exits) of each station. Closed schema: every field is required, null when the source has no value; any change of shape is a new format number. Generic on purpose (no network-specific identifier pattern).", + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "version", + "generatedAt", + "generator", + "source", + "license", + "attribution", + "modes", + "stops" + ], + "properties": { + "format": { + "description": "Version of the data contract.", + "const": 2 + }, + "version": { + "description": "UTC timestamp of the most recent source data change.", + "$ref": "#/$defs/timestamp" + }, + "generatedAt": { + "description": "UTC timestamp of the export.", + "$ref": "#/$defs/timestamp" + }, + "generator": { + "description": "Tool and version that produced the file, as name@version.", + "type": "string", + "pattern": "^[^@\\s]+@[^@\\s]+$" + }, + "source": { + "description": "Network and origin of the data.", + "type": "string", + "minLength": 1 + }, + "license": { + "description": "SPDX identifier of the licence of this database.", + "type": "string", + "minLength": 1 + }, + "attribution": { + "description": "Attribution required by the licence.", + "type": "string", + "minLength": 1 + }, + "modes": { + "description": "Modes kept at export time: a missing station of another mode was not exported, not absent from the network.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/mode" + } + }, + "stops": { + "type": "array", + "items": { + "$ref": "#/$defs/stop" + } + } + }, + "$defs": { + "timestamp": { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$" + }, + "mode": { + "enum": [ + "METRO", + "RER", + "TRAIN", + "TRAM", + "BUS", + "SHUTTLE", + "FUNICULAR", + "CABLEWAY" + ] + }, + "colour": { + "description": "Six uppercase hexadecimal digits without '#', or null.", + "type": [ + "string", + "null" + ], + "pattern": "^[0-9A-F]{6}$" + }, + "stop": { + "description": "A logical station (for IDFM: a zone de correspondance).", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "lat", + "lon", + "areas", + "lines", + "accesses" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "lat": { + "type": "number", + "minimum": -90, + "maximum": 90 + }, + "lon": { + "type": "number", + "minimum": -180, + "maximum": 180 + }, + "areas": { + "description": "Sub-areas queried by real-time services (for IDFM: stop areas).", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "lines": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/line" + } + }, + "accesses": { + "description": "Entrances and exits of the station's areas, sorted by id; empty when none is published.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/$defs/access" + } + } + } + }, + "line": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "label", + "mode", + "color", + "textColor" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "label": { + "type": "string", + "minLength": 1 + }, + "mode": { + "$ref": "#/$defs/mode" + }, + "color": { + "$ref": "#/$defs/colour" + }, + "textColor": { + "$ref": "#/$defs/colour" + } + } + }, + "access": { + "description": "An entrance and/or exit of a station.", + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "number", + "lat", + "lon", + "entry", + "exit" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "number": { + "description": "Number shown on the signage, or null.", + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "lat": { + "type": "number", + "minimum": -90, + "maximum": 90 + }, + "lon": { + "type": "number", + "minimum": -180, + "maximum": 180 + }, + "entry": { + "type": "boolean" + }, + "exit": { + "type": "boolean" + } + } + } + } +} diff --git a/pyproject.toml b/pyproject.toml index a06f10f..605f89b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ [project] name = "prim-api" # The repository tag (vX.Y.Z) must match this version: the release workflow checks it. -version = "1.0.0" +version = "1.1.0" description = "IDFM stations referential exporter, PRIM specs sync and Python clients, Opendatasoft datasets sync & validation" license = "MIT" requires-python = ">=3.12" diff --git a/tests/fixtures/referential/README.md b/tests/fixtures/referential/README.md index e764dfc..bc55a74 100644 --- a/tests/fixtures/referential/README.md +++ b/tests/fixtures/referential/README.md @@ -13,11 +13,12 @@ referential exporter (downloaded on 2026-09-27), limited to seven hubs | 66403 | Les Ambassadeurs / Norton | bus only | | 73848, 73849 | Funiculaire de Montmartre (top, bottom) | funicular, plus bus stop areas | -Every record of `zones-d-arrets`, `arrets`, `arrets-lignes` and -`referentiel-des-lignes` that belongs to these hubs is kept, so the extracts are -consistent (every reference resolves). +Every record of `zones-d-arrets`, `arrets`, `arrets-lignes`, +`referentiel-des-lignes`, `acces` and `relations-acces` (format 2) that belongs +to these hubs is kept, so the extracts are consistent (every reference resolves). Licences (see the README of the repository): `zones-de-correspondance`, -`zones-d-arrets` and `arrets` are under Licence Ouverte 2.0 (Etalab); +`zones-d-arrets`, `arrets`, `acces` and `relations-acces` are under Licence +Ouverte 2.0 (Etalab); `arrets-lignes` and `referentiel-des-lignes` are under ODbL. Source: Île-de-France Mobilités, https://data.iledefrance-mobilites.fr. diff --git a/tests/fixtures/referential/acces.jsonl b/tests/fixtures/referential/acces.jsonl new file mode 100644 index 0000000..933f17b --- /dev/null +++ b/tests/fixtures/referential/acces.jsonl @@ -0,0 +1,30 @@ +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:23:41+00:00", "accdescription": null, "accgeopoint": {"lat": 48.86168215645291, "lon": 2.3478155834385595}, "accid": "50148558", "accisentry": "true", "accisexit": "true", "accname": "Forum - Pte Lescot", "accprivatecode": "50148558", "accshortname": 3, "accversion": "1572433", "accxespg2154": 652152, "accyespg2154": 6862603, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:23:22+00:00", "accdescription": null, "accgeopoint": {"lat": 48.860433035639524, "lon": 2.3463455149117345}, "accid": "50148652", "accisentry": "true", "accisexit": "true", "accname": "Porte Marguerite de Navarre", "accprivatecode": "50148652", "accshortname": 1, "accversion": "1661194", "accxespg2154": 652043, "accyespg2154": 6862465, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:23:17+00:00", "accdescription": null, "accgeopoint": {"lat": 48.857836102563994, "lon": 2.3467460955207637}, "accid": "50148697", "accisentry": "true", "accisexit": "true", "accname": "Théâtre du Châtelet", "accprivatecode": "50148697", "accshortname": 15, "accversion": "1572335", "accxespg2154": 652070, "accyespg2154": 6862176, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:26:25+00:00", "accdescription": null, "accgeopoint": {"lat": 48.85765022656312, "lon": 2.3472936097922545}, "accid": "50147797", "accisentry": "true", "accisexit": "true", "accname": "pl. du Châtelet", "accprivatecode": "50147797", "accshortname": 16, "accversion": "1573087", "accxespg2154": 652110, "accyespg2154": 6862155, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-07-08T15:40:59+00:00", "acccreated": "2021-10-22T09:19:26+00:00", "accdescription": "Source IDFM", "accgeopoint": {"lat": 48.882125198505776, "lon": 2.3546502834548124}, "accid": "50170864", "accisentry": "true", "accisexit": "true", "accname": "r. de Maubeuge", "accprivatecode": "475328", "accshortname": 9, "accversion": "1821456", "accxespg2154": 652672, "accyespg2154": 6864872, "fournisseurid": "1", "fournisseurname": "SNCF"} +{"accchanged": "2026-04-03T08:54:13+00:00", "acccreated": "2021-10-22T08:22:44+00:00", "accdescription": null, "accgeopoint": {"lat": 48.89231028393414, "lon": 2.23764000536301}, "accid": "50148844", "accisentry": "true", "accisexit": "true", "accname": "Grande Arche", "accprivatecode": "50148844", "accshortname": 1, "accversion": "1572192", "accxespg2154": 644102, "accyespg2154": 6866081, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-04-03T08:54:13+00:00", "acccreated": "2021-10-22T08:23:43+00:00", "accdescription": null, "accgeopoint": {"lat": 48.89231028393414, "lon": 2.23764000536301}, "accid": "50148547", "accisentry": "true", "accisexit": "true", "accname": "salle des Colonnes", "accprivatecode": "50148547", "accshortname": 8, "accversion": "1572442", "accxespg2154": 644102, "accyespg2154": 6866081, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-04-03T08:54:13+00:00", "acccreated": "2021-10-22T08:25:54+00:00", "accdescription": null, "accgeopoint": {"lat": 48.893049404759196, "lon": 2.23790196337428}, "accid": "50147945", "accisentry": "true", "accisexit": "true", "accname": "pl. Carpeaux", "accprivatecode": "50147945", "accshortname": 7, "accversion": "1572962", "accxespg2154": 644122, "accyespg2154": 6866163, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:26:25+00:00", "accdescription": null, "accgeopoint": {"lat": 48.857121859953644, "lon": 2.349358276382533}, "accid": "50147798", "accisentry": "true", "accisexit": "true", "accname": "r. Saint-Martin", "accprivatecode": "50147798", "accshortname": 19, "accversion": "1573086", "accxespg2154": 652261, "accyespg2154": 6862095, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:26:26+00:00", "accdescription": null, "accgeopoint": {"lat": 48.858956863779206, "lon": 2.3461186959929123}, "accid": "50147794", "accisentry": "true", "accisexit": "true", "accname": "r. des Lavandières", "accprivatecode": "50147794", "accshortname": 13, "accversion": "1573090", "accxespg2154": 652025, "accyespg2154": 6862301, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:26:26+00:00", "accdescription": null, "accgeopoint": {"lat": 48.8575364397331, "lon": 2.347867476922391}, "accid": "50147796", "accisentry": "true", "accisexit": "true", "accname": "Théâtre de la Ville", "accprivatecode": "50147796", "accshortname": 17, "accversion": "1573088", "accxespg2154": 652152, "accyespg2154": 6862142, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:26:26+00:00", "accdescription": null, "accgeopoint": {"lat": 48.85919127439639, "lon": 2.3462247940508156}, "accid": "50147795", "accisentry": "true", "accisexit": "true", "accname": "r. de Rivoli", "accprivatecode": "50147795", "accshortname": 11, "accversion": "1573089", "accxespg2154": 652033, "accyespg2154": 6862327, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-07-08T15:50:35+00:00", "acccreated": "2021-10-22T09:21:09+00:00", "accdescription": "Source IDFM - Accès principal Gare", "accgeopoint": {"lat": 48.88006704376393, "lon": 2.354893954871231}, "accid": "50170424", "accisentry": "true", "accisexit": "true", "accname": "Esplanade Napoléon III", "accprivatecode": "475326", "accshortname": 5, "accversion": "1821459", "accxespg2154": 652688, "accyespg2154": 6864643, "fournisseurid": "1", "fournisseurname": "SNCF"} +{"accchanged": "2026-04-03T08:54:13+00:00", "acccreated": "2021-10-22T08:23:43+00:00", "accdescription": null, "accgeopoint": {"lat": 48.891664498635556, "lon": 2.2393270169352935}, "accid": "50148546", "accisentry": "true", "accisexit": "true", "accname": "Parvis - Esplanade", "accprivatecode": "50148546", "accshortname": 4, "accversion": "1572443", "accxespg2154": 644225, "accyespg2154": 6866008, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2024-05-29T16:08:43+00:00", "acccreated": "2021-10-22T08:26:27+00:00", "accdescription": null, "accgeopoint": {"lat": 48.86138627943126, "lon": 2.346333555008558}, "accid": "50147790", "accisentry": "true", "accisexit": "true", "accname": "Forum Porte Berger", "accprivatecode": "50147790", "accshortname": 2, "accversion": "1573094", "accxespg2154": 652043, "accyespg2154": 6862571, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:26:25+00:00", "accdescription": null, "accgeopoint": {"lat": 48.85729823340258, "lon": 2.3487154874139216}, "accid": "50147799", "accisentry": "true", "accisexit": "true", "accname": "av. Victoria", "accprivatecode": "50147799", "accshortname": 18, "accversion": "1573085", "accxespg2154": 652214, "accyespg2154": 6862115, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:26:27+00:00", "accdescription": null, "accgeopoint": {"lat": 48.85931983295799, "lon": 2.345064628040473}, "accid": "50147791", "accisentry": "true", "accisexit": "true", "accname": "r. Bertin Poirée", "accprivatecode": "50147791", "accshortname": 12, "accversion": "1573093", "accxespg2154": 651948, "accyespg2154": 6862342, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-07-08T15:51:15+00:00", "acccreated": "2021-10-22T09:21:40+00:00", "accdescription": "Source IDFM", "accgeopoint": {"lat": 48.88052973897501, "lon": 2.3539746400753887}, "accid": "50170276", "accisentry": "true", "accisexit": "true", "accname": "Cour des Taxis", "accprivatecode": "475319", "accshortname": 7, "accversion": "1821460", "accxespg2154": 652621, "accyespg2154": 6864695, "fournisseurid": "1", "fournisseurname": "SNCF"} +{"accchanged": "2026-04-03T08:54:13+00:00", "acccreated": "2021-10-22T08:22:45+00:00", "accdescription": null, "accgeopoint": {"lat": 48.89131214459331, "lon": 2.2390730318126426}, "accid": "50148843", "accisentry": "true", "accisexit": "true", "accname": "Boieldieu", "accprivatecode": "50148843", "accshortname": 3, "accversion": "1572193", "accxespg2154": 644206, "accyespg2154": 6865969, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-04-03T08:54:13+00:00", "acccreated": "2021-10-22T08:22:44+00:00", "accdescription": null, "accgeopoint": {"lat": 48.89198909247483, "lon": 2.2422955215511537}, "accid": "50148846", "accisentry": "true", "accisexit": "true", "accname": "Coupole - Regnault", "accprivatecode": "50148846", "accshortname": 6, "accversion": "1572190", "accxespg2154": 644443, "accyespg2154": 6866042, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2024-09-03T10:29:51+00:00", "acccreated": "2021-10-22T08:25:57+00:00", "accdescription": null, "accgeopoint": {"lat": 48.885727882381396, "lon": 2.342414209468209}, "accid": "50147931", "accisentry": "true", "accisexit": "true", "accname": "Entrée / Sortie", "accprivatecode": "50147931", "accshortname": 0, "accversion": "1607028", "accxespg2154": 651778, "accyespg2154": 6865280, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2024-05-29T16:08:50+00:00", "acccreated": "2021-10-22T08:26:30+00:00", "accdescription": null, "accgeopoint": {"lat": 48.87939408720543, "lon": 2.3568521346592783}, "accid": "50147779", "accisentry": "true", "accisexit": "true", "accname": "r. du Fbg Saint-Denis", "accprivatecode": "50147779", "accshortname": 1, "accversion": "1573104", "accxespg2154": 652831, "accyespg2154": 6864567, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2024-05-29T16:08:49+00:00", "acccreated": "2021-10-22T08:26:29+00:00", "accdescription": null, "accgeopoint": {"lat": 48.87975842480586, "lon": 2.3543659983381993}, "accid": "50147780", "accisentry": "true", "accisexit": "true", "accname": "bd de Denain", "accprivatecode": "50147780", "accshortname": 4, "accversion": "1573103", "accxespg2154": 652649, "accyespg2154": 6864609, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:26:26+00:00", "accdescription": null, "accgeopoint": {"lat": 48.859563926572946, "lon": 2.3469425129335297}, "accid": "50147793", "accisentry": "true", "accisexit": "true", "accname": "pl. Ste-Opportune", "accprivatecode": "50147793", "accshortname": 10, "accversion": "1573091", "accxespg2154": 652086, "accyespg2154": 6862368, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-08-21T13:52:24+00:00", "acccreated": "2021-10-22T08:26:27+00:00", "accdescription": null, "accgeopoint": {"lat": 48.858629633678966, "lon": 2.3471314117805697}, "accid": "50147792", "accisentry": "true", "accisexit": "true", "accname": "r. Saint-Denis", "accprivatecode": "50147792", "accshortname": 14, "accversion": "1573092", "accxespg2154": 652099, "accyespg2154": 6862264, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-04-03T08:54:13+00:00", "acccreated": "2021-10-22T08:22:44+00:00", "accdescription": null, "accgeopoint": {"lat": 48.89131976462272, "lon": 2.2402731149319735}, "accid": "50148845", "accisentry": "true", "accisexit": "true", "accname": "Calder-Miró", "accprivatecode": "50148845", "accshortname": 5, "accversion": "1572191", "accxespg2154": 644294, "accyespg2154": 6865969, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2024-09-03T10:29:51+00:00", "acccreated": "2021-10-22T08:25:57+00:00", "accdescription": null, "accgeopoint": {"lat": 48.88454137479853, "lon": 2.3425246490205707}, "accid": "50147930", "accisentry": "true", "accisexit": "true", "accname": "Entrée / Sortie", "accprivatecode": "50147930", "accshortname": 0, "accversion": "1607029", "accxespg2154": 651785, "accyespg2154": 6865148, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2024-05-29T16:08:51+00:00", "acccreated": "2021-10-22T08:26:30+00:00", "accdescription": null, "accgeopoint": {"lat": 48.87959674224068, "lon": 2.356072418504847}, "accid": "50147778", "accisentry": "true", "accisexit": "true", "accname": "r. de Saint-Quentin", "accprivatecode": "50147778", "accshortname": 2, "accversion": "1573105", "accxespg2154": 652774, "accyespg2154": 6864590, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2026-04-03T08:54:13+00:00", "acccreated": "2021-10-22T08:23:43+00:00", "accdescription": null, "accgeopoint": {"lat": 48.8917590491432, "lon": 2.2372252771552845}, "accid": "50148548", "accisentry": "true", "accisexit": "true", "accname": "Dôme - Valmy", "accprivatecode": "50148548", "accshortname": 2, "accversion": "1572441", "accxespg2154": 644071, "accyespg2154": 6866020, "fournisseurid": "59", "fournisseurname": "RATP"} +{"accchanged": "2024-05-29T16:08:48+00:00", "acccreated": "2021-10-22T08:26:29+00:00", "accdescription": null, "accgeopoint": {"lat": 48.879697680971624, "lon": 2.354775811992991}, "accid": "50147781", "accisentry": "true", "accisexit": "true", "accname": "r. de Dunkerque", "accprivatecode": "50147781", "accshortname": 3, "accversion": "1573102", "accxespg2154": 652679, "accyespg2154": 6864602, "fournisseurid": "59", "fournisseurname": "RATP"} diff --git a/tests/fixtures/referential/relations-acces.jsonl b/tests/fixtures/referential/relations-acces.jsonl new file mode 100644 index 0000000..c1e0259 --- /dev/null +++ b/tests/fixtures/referential/relations-acces.jsonl @@ -0,0 +1,44 @@ +{"accid": "50148844", "accversion": "1572192", "version": "1821228-1327660", "zdaid": "470548"} +{"accid": "50148547", "accversion": "1572442", "version": "1821228-1327660", "zdaid": "470548"} +{"accid": "50148844", "accversion": "1572192", "version": "1801046-1327660", "zdaid": "470549"} +{"accid": "50148846", "accversion": "1572190", "version": "1821228-1327660", "zdaid": "470548"} +{"accid": "50147790", "accversion": "1573094", "version": "1782229-1782252", "zdaid": "45102"} +{"accid": "50170276", "accversion": "1821460", "version": "1590890-1180723", "zdaid": "462394"} +{"accid": "50147781", "accversion": "1573102", "version": "1590890-1180723", "zdaid": "462394"} +{"accid": "50147931", "accversion": "1607028", "version": "45558-73848", "zdaid": "45558"} +{"accid": "50147781", "accversion": "1573102", "version": "1090149-1180723", "zdaid": "462941"} +{"accid": "50147780", "accversion": "1573103", "version": "1090149-1180723", "zdaid": "462941"} +{"accid": "50147798", "accversion": "1573086", "version": "1641337-71264", "zdaid": "42587"} +{"accid": "50147797", "accversion": "1573087", "version": "1641337-71264", "zdaid": "42587"} +{"accid": "50147792", "accversion": "1573092", "version": "1641337-71264", "zdaid": "42587"} +{"accid": "50148546", "accversion": "1572443", "version": "1821228-1327660", "zdaid": "470548"} +{"accid": "50148843", "accversion": "1572193", "version": "1801046-1327660", "zdaid": "470549"} +{"accid": "50147945", "accversion": "1572962", "version": "1801046-1327660", "zdaid": "470549"} +{"accid": "50148652", "accversion": "1661194", "version": "1782229-1782252", "zdaid": "45102"} +{"accid": "50147799", "accversion": "1573085", "version": "1641337-71264", "zdaid": "42587"} +{"accid": "50148697", "accversion": "1572335", "version": "1641337-71264", "zdaid": "42587"} +{"accid": "50147945", "accversion": "1572962", "version": "1821228-1327660", "zdaid": "470548"} +{"accid": "50148558", "accversion": "1572433", "version": "1782229-1782252", "zdaid": "45102"} +{"accid": "50147780", "accversion": "1573103", "version": "1590890-1180723", "zdaid": "462394"} +{"accid": "50147778", "accversion": "1573105", "version": "1090149-1180723", "zdaid": "462941"} +{"accid": "50147779", "accversion": "1573104", "version": "1090149-1180723", "zdaid": "462941"} +{"accid": "50147930", "accversion": "1607029", "version": "45559-73849", "zdaid": "45559"} +{"accid": "50148845", "accversion": "1572191", "version": "1821228-1327660", "zdaid": "470548"} +{"accid": "50148548", "accversion": "1572441", "version": "1821228-1327660", "zdaid": "470548"} +{"accid": "50148546", "accversion": "1572443", "version": "1801046-1327660", "zdaid": "470549"} +{"accid": "50148845", "accversion": "1572191", "version": "1801046-1327660", "zdaid": "470549"} +{"accid": "50148547", "accversion": "1572442", "version": "1801046-1327660", "zdaid": "470549"} +{"accid": "50148846", "accversion": "1572190", "version": "1801046-1327660", "zdaid": "470549"} +{"accid": "50170864", "accversion": "1821456", "version": "1590890-1180723", "zdaid": "462394"} +{"accid": "50147779", "accversion": "1573104", "version": "1590890-1180723", "zdaid": "462394"} +{"accid": "50148558", "accversion": "1572433", "version": "1641337-71264", "zdaid": "42587"} +{"accid": "50147794", "accversion": "1573090", "version": "1641337-71264", "zdaid": "42587"} +{"accid": "50147795", "accversion": "1573089", "version": "1641337-71264", "zdaid": "42587"} +{"accid": "50147791", "accversion": "1573093", "version": "1641337-71264", "zdaid": "42587"} +{"accid": "50148843", "accversion": "1572193", "version": "1821228-1327660", "zdaid": "470548"} +{"accid": "50148548", "accversion": "1572441", "version": "1801046-1327660", "zdaid": "470549"} +{"accid": "50147778", "accversion": "1573105", "version": "1590890-1180723", "zdaid": "462394"} +{"accid": "50170424", "accversion": "1821459", "version": "1590890-1180723", "zdaid": "462394"} +{"accid": "50148652", "accversion": "1661194", "version": "1641337-71264", "zdaid": "42587"} +{"accid": "50147793", "accversion": "1573091", "version": "1641337-71264", "zdaid": "42587"} +{"accid": "50147796", "accversion": "1573088", "version": "1641337-71264", "zdaid": "42587"} diff --git a/tests/test_referential.py b/tests/test_referential.py index 97ae074..fcc2438 100644 --- a/tests/test_referential.py +++ b/tests/test_referential.py @@ -483,8 +483,8 @@ def test_schema_is_valid_draft_2020_12(self): assert json.loads(schema_text(1)) == load_schema(1) def test_unknown_format(self): - with pytest.raises(ValueError, match="format 2"): - schema_filename(2) + with pytest.raises(ValueError, match="format 3"): + schema_filename(3) @pytest.mark.parametrize( "mutate", @@ -579,9 +579,9 @@ def test_bad_mode(self, data_dir): assert "unknown mode" in result.output def test_unknown_format(self, data_dir): - result = self.invoke("--no-download", "--data-dir", str(data_dir), "--format", "2") + result = self.invoke("--no-download", "--data-dir", str(data_dir), "--format", "3") assert result.exit_code == 1 - assert "format 2" in result.output + assert "format 3" in result.output @respx.mock def test_download_records_data_processed(self, tmp_path): diff --git a/tests/test_referential_format2.py b/tests/test_referential_format2.py new file mode 100644 index 0000000..f457026 --- /dev/null +++ b/tests/test_referential_format2.py @@ -0,0 +1,282 @@ +"""Tests for format 2 of the stations referential (``prim_api.referential.format2``). + +Format 2 = format 1 + the accesses of each station. Every rule of format 2 has +its own test, on the same real extracts as format 1 plus the matching records of +``acces`` and ``relations-acces`` (``tests/fixtures/referential/``). +""" + +from __future__ import annotations + +import copy +import json +import shutil +from datetime import UTC, datetime +from pathlib import Path + +import jsonschema +import pytest +from typer.testing import CliRunner + +import prim_api.datasets as ds_mod +from prim_api.referential import cli as cli_mod +from prim_api.referential.format1 import build_stations_file, lambert93_to_wgs84 +from prim_api.referential.format2 import ( + ACCES, + RELATIONS_ACCES, + SOURCE_DATASETS, + ReferentialError, + build_stations_file_format2, +) +from prim_api.referential.schema import load_schema, validate_stations_file + +FIXTURES = Path(__file__).parent / "fixtures" / "referential" +PARIS_MODES = ("METRO", "RER", "TRAIN", "TRAM") +GENERATED_AT = datetime(2026, 9, 28, 9, 0, 0, tzinfo=UTC) +DATA_PROCESSED = { + "zones-de-correspondance": "2026-09-27T00:01:06+00:00", + "zones-d-arrets": "2026-09-25T00:40:57+00:00", + "arrets": "2026-09-26T23:41:25+00:00", + "arrets-lignes": "2026-09-26T15:06:36+00:00", + "referentiel-des-lignes": "2026-09-27T07:05:39+00:00", + ACCES: "2026-09-28T01:10:00+00:00", + RELATIONS_ACCES: "2026-09-28T01:12:00+00:00", +} + + +def _load(dataset_id: str) -> list[dict]: + text = (FIXTURES / f"{dataset_id}.jsonl").read_text(encoding="utf-8") + return [json.loads(line) for line in text.splitlines() if line] + + +@pytest.fixture(scope="module") +def records() -> dict[str, list[dict]]: + return {dataset_id: _load(dataset_id) for dataset_id in SOURCE_DATASETS} + + +@pytest.fixture(scope="module") +def converter(): + return lambert93_to_wgs84() + + +def build(records, converter, modes=PARIS_MODES, **overrides): + arguments = { + "modes": modes, + "data_processed": DATA_PROCESSED, + "generator": "idfm-prim-api@1.1.0", + "generated_at": GENERATED_AT, + "to_wgs84": converter, + } + arguments.update(overrides) + return build_stations_file_format2(records, **arguments) + + +def stop_by_id(document: dict, stop_id: str) -> dict: + return next(stop for stop in document["stops"] if stop["id"] == stop_id) + + +class TestHeader: + def test_format_and_version_over_seven_datasets(self, records, converter): + document = build(records, converter) + assert document["format"] == 2 + # The most recent of the seven data_processed dates: relations-acces. + assert document["version"] == "2026-09-28T01:12:00Z" + + def test_same_stations_as_format_1(self, records, converter): + format2 = build(records, converter) + format1 = build_stations_file( + records, + modes=PARIS_MODES, + data_processed=DATA_PROCESSED, + generator="idfm-prim-api@1.1.0", + generated_at=GENERATED_AT, + to_wgs84=converter, + ) + stripped = [{k: v for k, v in stop.items() if k != "accesses"} for stop in format2["stops"]] + assert stripped == format1["stops"] + + +class TestAccesses: + def test_fields_of_an_access(self, records, converter): + halles = stop_by_id(build(records, converter), "IDFM:474151") + assert halles["accesses"] == [ + { + "id": "IDFM:50147790", + "name": "Forum Porte Berger", + "number": 2, + "lat": 48.861386, + "lon": 2.346334, + "entry": True, + "exit": True, + }, + { + "id": "IDFM:50148558", + "name": "Forum - Pte Lescot", + "number": 3, + "lat": 48.861682, + "lon": 2.347816, + "entry": True, + "exit": True, + }, + { + "id": "IDFM:50148652", + "name": "Porte Marguerite de Navarre", + "number": 1, + "lat": 48.860433, + "lon": 2.346346, + "entry": True, + "exit": True, + }, + ] + + def test_counts_per_station(self, records, converter): + document = build(records, converter) + counts = {stop["name"]: len(stop["accesses"]) for stop in document["stops"]} + assert counts == { + "Châtelet - Les Halles": 3, + "Châtelet": 12, + "Gare du Nord": 7, + "La Défense": 8, + } + + def test_access_linked_to_two_areas_listed_once_and_sorted(self, records, converter): + for stop in build(records, converter)["stops"]: + ids = [access["id"] for access in stop["accesses"]] + assert len(ids) == len(set(ids)) + numeric = [int(value.removeprefix("IDFM:")) for value in ids] + assert numeric == sorted(numeric) + + def test_only_areas_of_selected_modes(self, records, converter): + # La Défense in tram only: its tram stop area has no published access. + defense = stop_by_id(build(records, converter, modes=("TRAM",)), "IDFM:71517") + assert defense["accesses"] == [] + + def test_number_may_be_null(self, records, converter): + altered = copy.deepcopy(records) + for access in altered[ACCES]: + if access["accid"] == "50147790": + access["accshortname"] = None + halles = stop_by_id(build(altered, converter), "IDFM:474151") + assert halles["accesses"][0]["number"] is None + validate_stations_file(build(altered, converter), 2) + + def test_exit_only_access_kept_with_flags(self, records, converter): + altered = copy.deepcopy(records) + for access in altered[ACCES]: + if access["accid"] == "50147790": + access["accisentry"] = "false" + first = stop_by_id(build(altered, converter), "IDFM:474151")["accesses"][0] + assert (first["entry"], first["exit"]) == (False, True) + + +class TestSourceErrors: + def _alter(self, records, dataset_id, change): + altered = copy.deepcopy(records) + change(altered[dataset_id]) + return altered + + def test_missing_access_datasets(self, records, converter): + partial = {k: v for k, v in records.items() if k != ACCES} + with pytest.raises(ReferentialError, match="missing source dataset"): + build(partial, converter) + + def test_relation_to_unknown_access(self, records, converter): + def drop(rows): + rows[:] = [a for a in rows if a["accid"] != "50147790"] + + with pytest.raises(ReferentialError, match="50147790"): + build(self._alter(records, ACCES, drop), converter) + + @pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("accname", " ", "name"), + ("accgeopoint", None, "coordinates"), + ("accisentry", "yes", "invalid accisentry"), + ("accshortname", "2", "non-integer"), + ], + ) + def test_invalid_access(self, records, converter, field, value, message): + def change(rows): + for access in rows: + if access["accid"] == "50147790": + access[field] = value + + with pytest.raises(ReferentialError, match=message): + build(self._alter(records, ACCES, change), converter) + + def test_duplicate_access_id(self, records, converter): + def duplicate(rows): + rows.append(dict(rows[0])) + + with pytest.raises(ReferentialError, match="duplicate access id"): + build(self._alter(records, ACCES, duplicate), converter) + + +class TestSchema: + @pytest.fixture + def document(self, records, converter): + return build(records, converter) + + def test_valid_and_closed(self, document): + jsonschema.Draft202012Validator.check_schema(load_schema(2)) + validate_stations_file(document, 2) + + @pytest.mark.parametrize( + "mutate", + [ + lambda d: d.update(format=1), + lambda d: d["stops"][0].pop("accesses"), + lambda d: d["stops"][0]["accesses"][0].update(wheelchair=True), + lambda d: d["stops"][0]["accesses"][0].pop("number"), + lambda d: d["stops"][0]["accesses"][0].update(entry="true"), + lambda d: d["stops"][0]["accesses"][0].update(number=-1), + ], + ) + def test_rejects(self, document, mutate): + mutate(document) + with pytest.raises(jsonschema.ValidationError): + validate_stations_file(document, 2) + + def test_format_1_document_is_not_format_2(self, records, converter): + format1 = build_stations_file( + records, + modes=PARIS_MODES, + data_processed=DATA_PROCESSED, + generator="idfm-prim-api@1.1.0", + generated_at=GENERATED_AT, + to_wgs84=converter, + ) + with pytest.raises(jsonschema.ValidationError): + validate_stations_file(format1, 2) + + +class TestCli: + runner = CliRunner() + + @pytest.fixture + def data_dir(self, tmp_path) -> Path: + for dataset_id in SOURCE_DATASETS: + shutil.copy(FIXTURES / f"{dataset_id}.jsonl", tmp_path / f"{dataset_id}.jsonl") + ds_mod.record_metadata_fields( + dataset_id, {"data_processed": DATA_PROCESSED[dataset_id]}, tmp_path + ) + return tmp_path + + def test_exports_format_2(self, data_dir, tmp_path): + output = tmp_path / "stations.json" + result = self.runner.invoke( + cli_mod.app, + ["--no-download", "--data-dir", str(data_dir), "--format", "2", + "--modes", "METRO,RER,TRAIN,TRAM", "--output", str(output)], + ) # fmt: skip + assert result.exit_code == 0, result.output + document = json.loads(output.read_text(encoding="utf-8")) + validate_stations_file(document, 2) + assert "format 2" in result.output and "30 accesses" in result.output + + def test_format_1_still_produced(self, data_dir): + result = self.runner.invoke( + cli_mod.app, ["--no-download", "--data-dir", str(data_dir), "--format", "1"] + ) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["format"] == 1