From b9d1700b4214dd629e61af95d5563063c45a1962 Mon Sep 17 00:00:00 2001 From: Jay Ravani Date: Mon, 26 Oct 2026 11:55:54 +0100 Subject: [PATCH 1/2] feat: skip unchanged dataset folders on a whole-folder import -import-data and -create-db import every dataset folder whole, and that import was neither checked nor recorded, so each run re-ran citydb-tool over every file of every folder. --import-mode=skip kept it free of duplicates, but a scheduled import that only has to pick up new files re-read everything each time. A whole-folder import is now recorded by the fingerprint of the folder's files, as bbox imports already were, with an extent covering the whole world: it holds every feature, so it covers any bbox in any mode. A later whole-folder import, or a bbox import, of unchanged files skips citydb-tool; adding, replacing or removing a file changes the fingerprint and the folder is imported again. An import with IMPORT_LIMIT set stays unrecorded, as before, and so does one without a database connection, as in unit tests. docs/deployment/adding-data.md describes the scheduled import, the staging of new files by rename, and that one invalid attribution.json stops the country's whole import. --- CHANGELOG.md | 11 +++++ docs/deployment/adding-data.md | 43 +++++++++++++++++++ internal/importer/extent.go | 31 ++++++++----- .../import_extent_integration_test.go | 33 ++++++++++++++ mkdocs.yml | 1 + 5 files changed, 108 insertions(+), 11 deletions(-) create mode 100644 docs/deployment/adding-data.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c7eca6d..3e5fb7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,17 @@ Each release is listed with its breaking changes first. Releases before v0.8.0 are described in their tag messages. +## Unreleased + +Features: +- -import-data and -create-db record a whole-folder import by its + files' fingerprint, as bbox imports already were, so a later import + skips citydb-tool for every dataset folder whose files are unchanged. + A scheduled `c2t -import-data` is then cheap when nothing is new. A + bbox run after a whole-folder import of unchanged files is skipped + too. docs/deployment/adding-data.md covers the scheduled import, the + staging of new files and the attribution check. + ## v0.13.0 (2026-10-25) Features: diff --git a/docs/deployment/adding-data.md b/docs/deployment/adding-data.md new file mode 100644 index 0000000..a93800e --- /dev/null +++ b/docs/deployment/adding-data.md @@ -0,0 +1,43 @@ +--- +audience: developer +--- + +# Adding data to a running deployment + +A country's database grows by adding source files under `data/lod2//` and importing them. The import can run on a schedule: a run over unchanged files does nothing. + +```bash +COUNTRY=germany c2t -create-db # once per country: creates the database and imports every folder +COUNTRY=germany c2t -import-data # afterwards, as often as needed: imports what changed +``` + +`-import-data` only imports. Feature extraction and the PyLovo link run when a region is requested (`POST /api/v1/runs`), or with `-extract-features` and `-link-pylovo`. + +## What an import does + +| Folder state since the last import | Result | +|---|---| +| No files added, replaced or removed | citydb-tool is not run for the folder | +| Files added or replaced | The folder is imported again; buildings already in the database are skipped, so none is duplicated | +| New dataset folder | The folder is imported | + +A folder counts as changed when the path, size or modification time of any file in its `gml/` or `cityjson/` folder changes. A run for a bbox (`POST /api/v1/runs`, or `-import-data -bbox`) after a whole-folder import of unchanged files does not run citydb-tool either. + +!!! warning "One invalid attribution file stops the country's import" + Before anything is imported, every dataset folder of the country is checked: it needs a valid `attribution.json` ([Dataset attribution file](../code/attribution/index.md)) and a `gml/` or `cityjson/` folder, and no model file may lie directly in `data/lod2//`. One failing folder stops the whole import for that country, with an error naming the folder; nothing is imported until it is fixed. A scheduled job should report the error. + +## Staging files + +A file that is still being written must not be in a model folder when an import runs. Write files outside the model folders, then rename them into place on the same filesystem, where a rename is atomic. No marker file is needed. + +| Adding | Steps | +|---|---| +| A new dataset | Write the folder under a hidden name, e.g. `data/lod2/germany/.bavaria-upload/` with `attribution.json` and `gml/`, then rename it to `data/lod2/germany/bavaria/`. Hidden folders are not read. | +| Files to an existing dataset | Write each file to `data/lod2/germany/.staging/`, then rename it into `data/lod2/germany/bavaria/gml/`. | + +A file renamed into place during an import is picked up by the next one. + +## Limitations + +- A provider's corrected tile is not applied: buildings already in the database are skipped on re-import, and their extracted features are kept. +- An import with `IMPORT_LIMIT` set is partial by design and is never skipped. diff --git a/internal/importer/extent.go b/internal/importer/extent.go index 1968d17..d2527f6 100644 --- a/internal/importer/extent.go +++ b/internal/importer/extent.go @@ -148,34 +148,43 @@ func recordExtent(conn *pgxpool.Pool, d Dataset, fingerprint string, b importBbo return nil } -// importDatasetOnce imports d for bbox unless an earlier import already -// covered it, and records the bbox once the import succeeds. A whole-folder -// import (empty bbox) or a limited one (partial by design) is neither skipped -// nor recorded. +// wholeFolder is the extent a whole-folder import (empty bbox) records: it holds +// every feature of the folder, so it covers any later bbox in any mode. +var wholeFolder = importBbox{xmin: -180, ymin: -90, xmax: 180, ymax: 90, srid: 4326} + +// importDatasetOnce imports d for bbox unless an earlier import from the same +// source files already covered it, and records the extent once the import +// succeeds. An empty bbox imports the whole folder and records wholeFolder. A +// limited import (partial by design) is neither skipped nor recorded, nor is any +// import when conn is nil, as in tests that never reach a real database. func importDatasetOnce(conn *pgxpool.Pool, cityDBExecPath string, d Dataset, cfg *config.Config, bbox, bboxMode string) error { - if bbox == "" || cfg.CityDB.ImportLimit > 0 { + if cfg.CityDB.ImportLimit > 0 || conn == nil { return importCityDBFiles(cityDBExecPath, d, cfg, bbox, bboxMode) } - b, err := parseImportBbox(bbox, cfg.CityDB.SRID) - if err != nil { - return err + b, mode := wholeFolder, "intersects" + if bbox != "" { + parsed, err := parseImportBbox(bbox, cfg.CityDB.SRID) + if err != nil { + return err + } + b, mode = parsed, bboxMode } fingerprint, err := inputFingerprint(d) if err != nil { return err } - covered, err := extentImported(conn, d, fingerprint, b, bboxMode) + covered, err := extentImported(conn, d, fingerprint, b, mode) if err != nil { return err } if covered { - utils.Info.Printf("%s already imported for bbox %s, skipping citydb-tool", d.Attribution.DatasetID, bbox) + utils.Info.Printf("%s already imported from unchanged files for bbox %q, skipping citydb-tool", d.Attribution.DatasetID, bbox) return nil } if err := importCityDBFiles(cityDBExecPath, d, cfg, bbox, bboxMode); err != nil { return err } - return recordExtent(conn, d, fingerprint, b, bboxMode) + return recordExtent(conn, d, fingerprint, b, mode) } diff --git a/internal/importer/import_extent_integration_test.go b/internal/importer/import_extent_integration_test.go index 9fd5761..6d27106 100644 --- a/internal/importer/import_extent_integration_test.go +++ b/internal/importer/import_extent_integration_test.go @@ -117,3 +117,36 @@ func TestImportCityDBData_ImportLimitIsNotRecorded(t *testing.T) { t.Fatalf("citydb import called %d times, want 2 (a limited import is partial)", got) } } + +// TestImportCityDBData_SkipsUnchangedWholeFolder pins that a whole-folder import +// (no bbox), as a scheduled run makes, is recorded: repeating it, or a bbox +// import afterwards, skips citydb-tool until the folder's files change. +func TestImportCityDBData_SkipsUnchangedWholeFolder(t *testing.T) { + ctx := context.Background() + cfg, logPath, gml := extentConfig(t, ctx, "extent_whole_lod2") + + steps := []struct { + name, bbox, mode string + addFile string + wantCalls int + }{ + {name: "first whole-folder import", wantCalls: 1}, + {name: "same files again", wantCalls: 1}, + {name: "bbox import after a whole-folder one", bbox: "16.349,48.210,16.351,48.212,4326", mode: "intersects", wantCalls: 1}, + {name: "a file added", addFile: "tile-2.gml", wantCalls: 2}, + {name: "same files again after the change", wantCalls: 2}, + } + for _, s := range steps { + if s.addFile != "" { + if err := os.WriteFile(filepath.Join(gml, s.addFile), []byte(""), 0o644); err != nil { + t.Fatalf("%s: %v", s.name, err) + } + } + if err := importer.ImportCityDBData(testPool, cfg, s.bbox, s.mode); err != nil { + t.Fatalf("%s: ImportCityDBData: %v", s.name, err) + } + if got := importCalls(t, logPath); got != s.wantCalls { + t.Fatalf("%s: citydb import called %d times in total, want %d", s.name, got, s.wantCalls) + } + } +} diff --git a/mkdocs.yml b/mkdocs.yml index 1ea6926..00c6a2d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -116,6 +116,7 @@ nav: - Home: index.md - Setup & Installation: installation/setup.md - Deployment Guide: deployment/index.md + - Adding Data: deployment/adding-data.md - Configuration: configuration/index.md - Troubleshooting: troubleshooting/index.md - API reference: api.md From caf1941c54a57cfcb8d0edbcdfb0f48548412bf4 Mon Sep 17 00:00:00 2001 From: Jay Ravani Date: Mon, 26 Oct 2026 11:57:50 +0100 Subject: [PATCH 2/2] feat: check a staged dataset folder before publishing it A data folder was only checked when it was imported, so files with a broken attribution.json or a CRS City2TABULA does not use reached the data folder before anyone found out, and the attribution rules lived nowhere a publishing step could run them. c2t -check-data checks a staged dataset folder for COUNTRY without importing it or connecting to a database. It runs the import's own checks (attribution.json, a gml/ or cityjson/ folder) and adds: no model file outside those folders, hidden ones skipped as the import skips them; no published folder of the country using the dataset_id with a different attribution.json (the same one means the staged files extend that dataset); and for every model file, gzipped or not and in any subfolder, that it is CityGML or CityJSON and that its declared CRS (srsName in the first MiB, or the CityJSON metadata.referenceSystem) resolves to the SRID the import uses. Accepted CRS names are the EPSG urn, OGC URI and EPSG: forms, the AdV names for UTM 32 and 33, and the compound EPSG:7415 for RD New with NAP. Anything else, such as "unknown" or a misspelt urn, fails. The command lists every problem with its path and exits 1, or prints "ready to publish" and exits 0. Run on real folders: Bavaria, NRW, 3DBAG, Vienna and Brno pass; Prague fails on its missing attribution.json and srsName "unknown". --- CHANGELOG.md | 6 + cmd/c2t/main.go | 20 +++ docs/deployment/adding-data.md | 19 +++ internal/flags/flags.go | 2 + internal/importer/check.go | 279 ++++++++++++++++++++++++++++++++ internal/importer/check_test.go | 166 +++++++++++++++++++ 6 files changed, 492 insertions(+) create mode 100644 internal/importer/check.go create mode 100644 internal/importer/check_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e5fb7a..b86d942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ Features: bbox run after a whole-folder import of unchanged files is skipped too. docs/deployment/adding-data.md covers the scheduled import, the staging of new files and the attribution check. +- c2t -check-data checks a staged dataset folder for COUNTRY + without importing it or connecting to a database: attribution.json, + the gml/ and cityjson/ folders, a dataset_id no published folder uses + with another attribution, and each model file's format and declared + CRS, which must be the SRID the import uses. It lists every problem + and exits with status 1, or prints "ready to publish". ## v0.13.0 (2026-10-25) diff --git a/cmd/c2t/main.go b/cmd/c2t/main.go index 4b27d4e..3ca59d1 100644 --- a/cmd/c2t/main.go +++ b/cmd/c2t/main.go @@ -42,6 +42,11 @@ func main() { utils.InitLogger() config := config.LoadConfig() + // Before Validate: a check needs only COUNTRY, not database settings. + if f.CheckData != "" { + os.Exit(checkData(&config, f.CheckData)) + } + if err := config.Validate(); err != nil { utils.Error.Fatal("Invalid configuration:", err) } @@ -122,3 +127,18 @@ func main() { utils.Info.Println(flagMessages.LinkPylovo.Success) } } + +// checkData prints every problem importer.CheckDataset finds in dir and returns +// the process exit code: 0 when the folder is ready to publish, 1 otherwise. +func checkData(cfg *config.Config, dir string) int { + problems := importer.CheckDataset(cfg, dir) + for _, p := range problems { + fmt.Fprintln(os.Stderr, p) + } + if len(problems) > 0 { + fmt.Fprintf(os.Stderr, "%s: %d problem(s); not ready to publish under %s\n", dir, len(problems), cfg.Country) + return 1 + } + fmt.Printf("%s: ready to publish under %s\n", dir, cfg.Country) + return 0 +} diff --git a/docs/deployment/adding-data.md b/docs/deployment/adding-data.md index a93800e..f309d09 100644 --- a/docs/deployment/adding-data.md +++ b/docs/deployment/adding-data.md @@ -26,6 +26,25 @@ A folder counts as changed when the path, size or modification time of any file !!! warning "One invalid attribution file stops the country's import" Before anything is imported, every dataset folder of the country is checked: it needs a valid `attribution.json` ([Dataset attribution file](../code/attribution/index.md)) and a `gml/` or `cityjson/` folder, and no model file may lie directly in `data/lod2//`. One failing folder stops the whole import for that country, with an error naming the folder; nothing is imported until it is fixed. A scheduled job should report the error. +## Checking files before publishing + +`-check-data` checks a staged dataset folder for the country without importing it or connecting to a database. It needs only `COUNTRY`. + +```bash +COUNTRY=germany c2t -check-data data/lod2/germany/.bavaria-upload +``` + +| Check | Fails when | +|---|---| +| `attribution.json` | It is missing or breaks a rule of the [attribution file](../code/attribution/index.md) | +| Model folders | There is neither a `gml/` nor a `cityjson/` folder, or a model file lies outside them (hidden folders are not read) | +| `dataset_id` | A published folder of the country uses it with a different `attribution.json`. The same file means the staged files extend that dataset | +| Model files | A file, gzipped or not and in any subfolder, is not CityGML or CityJSON, or its declared CRS is not the SRID the import uses: `CITYDB_SRID`, or the country's default ([Configuration](../configuration/index.md)) | + +The CRS is read from a CityGML file's `srsName` values in its first MiB and from a CityJSON file's `metadata.referenceSystem`. EPSG names are accepted in the `urn:ogc:def:crs:EPSG::`, `https://www.opengis.net/def/crs/EPSG/0/` and `EPSG:` forms, as are the AdV names `urn:adv:crs:ETRS89_UTM32*` and `urn:adv:crs:ETRS89_UTM33*` and the compound EPSG:7415 (Amersfoort / RD New + NAP). Any other name, such as `unknown`, fails. + +The command lists every problem, each naming its file or folder, and exits with status 1; a folder that passes prints `ready to publish` and exits with status 0. + ## Staging files A file that is still being written must not be in a model folder when an import runs. Write files outside the model folders, then rename them into place on the same filesystem, where a rename is atomic. No marker file is needed. diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 8b4fa9a..a151b12 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -14,6 +14,7 @@ type Flags struct { ShowVersion bool ShowV bool Bbox string + CheckData string BboxMode string } @@ -30,6 +31,7 @@ func ParseFlags() *Flags { flag.BoolVar(&f.ShowVersion, "version", false, "print version and exit") flag.BoolVar(&f.ShowV, "v", false, "print version and exit (shorthand)") flag.StringVar(&f.Bbox, "bbox", "", "Bounding box spatial filter for -import-data, format xmin,ymin,xmax,ymax[,srid] (passed through to citydb-tool)") + flag.StringVar(&f.CheckData, "check-data", "", "Check a staged dataset folder for publishing under COUNTRY (attribution.json, model folders, dataset_id, each model file's format and CRS) without importing it or connecting to a database; exits 1 listing every problem") flag.StringVar(&f.BboxMode, "bbox-mode", "intersects", "Bounding box filter mode for -bbox: intersects, contains, or on_tile") flag.Parse() return f diff --git a/internal/importer/check.go b/internal/importer/check.go new file mode 100644 index 0000000..ed7c06b --- /dev/null +++ b/internal/importer/check.go @@ -0,0 +1,279 @@ +package importer + +import ( + "bufio" + "compress/gzip" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/thd-spatial-ai/city2tabula/internal/attribution" + "github.com/thd-spatial-ai/city2tabula/internal/config" +) + +// gmlHeaderBytes is how much of a CityGML file is searched for its root element +// and srsName; providers declare the CRS on the first envelope or geometry. +const gmlHeaderBytes = 1 << 20 + +var ( + srsNamePattern = regexp.MustCompile(`srsName="([^"]*)"`) + referenceSystemPattern = regexp.MustCompile(`"referenceSystem"\s*:\s*"([^"]*)"`) + // epsgName matches the CRS name forms citydb-tool reads as an EPSG code: + // urn:ogc:def:crs:EPSG:: (also the compound urn:ogc:def:crs,crs:EPSG::...), + // the OGC http(s) URI and EPSG:. The first code is the horizontal CRS. + epsgName = regexp.MustCompile(`^(?:urn:ogc:def:crs(?:,crs)?:EPSG:[^:]*:|https?://www\.opengis\.net/def/crs/EPSG/0/|EPSG:)(\d+)`) +) + +// advCRSPrefixes maps the German AdV CRS names, which continue with a height +// system (e.g. "*DE_DHHN2016_NH"), to their horizontal EPSG code. +var advCRSPrefixes = map[string]int{ + "urn:adv:crs:ETRS89_UTM32": 25832, + "urn:adv:crs:ETRS89_UTM33": 25833, +} + +// compoundEPSG maps compound EPSG codes to their horizontal component. +var compoundEPSG = map[int]int{ + 7415: 28992, // Amersfoort / RD New + NAP height +} + +// resolveCRS returns the horizontal EPSG code a declared CRS name stands for. +func resolveCRS(name string) (int, error) { + for prefix, code := range advCRSPrefixes { + if strings.HasPrefix(name, prefix) { + return code, nil + } + } + m := epsgName.FindStringSubmatch(name) + if m == nil { + return 0, fmt.Errorf("CRS %q is not a recognised EPSG or AdV name", name) + } + code, _ := strconv.Atoi(m[1]) // \d+ always parses + if horizontal, ok := compoundEPSG[code]; ok { + return horizontal, nil + } + return code, nil +} + +// CheckDataset checks a staged dataset folder for publishing under cfg's country +// without importing it or touching a database: the rules the import applies +// (attribution.json, model folders, a dataset_id that clashes with no published +// folder of other content) plus each model file's format and declared CRS, which +// must be the SRID the import uses: CITYDB_SRID, or the country's default. It +// returns every problem, each naming its path. +func CheckDataset(cfg *config.Config, dir string) []error { + sridText, _, err := config.SRIDForCountry(cfg.Country) + if err != nil { + return []error{err} + } + if cfg.CityDB != nil && cfg.CityDB.SRID != "" { + sridText = cfg.CityDB.SRID + } + srid, err := strconv.Atoi(sridText) + if err != nil { + return []error{fmt.Errorf("SRID %q is not an EPSG code", sridText)} + } + + var problems []error + a, err := attribution.Load(dir) + if err != nil { + problems = append(problems, err) + } else if err := checkDatasetIDFree(cfg, dir, a); err != nil { + problems = append(problems, err) + } + problems = append(problems, strayModelFiles(dir)...) + inputs, err := datasetInputs(dir) + if err != nil { + return append(problems, err) + } + for _, in := range inputs { + problems = append(problems, checkModelFolder(in, srid)...) + } + return problems +} + +// checkDatasetIDFree refuses a dataset_id that a published folder already uses +// with a different attribution. The same attribution means the staged files +// extend that dataset. +func checkDatasetIDFree(cfg *config.Config, dir string, a attribution.Attribution) error { + published, err := DiscoverDatasets(cfg) + if err != nil { + return fmt.Errorf("published data for %s cannot be read: %w", cfg.Country, err) + } + staged, _ := filepath.Abs(dir) + for _, d := range published { + other, _ := filepath.Abs(d.Dir) + if d.Attribution.DatasetID == a.DatasetID && other != staged && d.Attribution != a { + return fmt.Errorf("%s: dataset_id %q is already used by %s with a different attribution.json", dir, a.DatasetID, d.Dir) + } + } + return nil +} + +// strayModelFiles reports model files outside gml/ and cityjson/, which an +// import would not read. Hidden entries are skipped, as the import skips them. +func strayModelFiles(dir string) []error { + var problems []error + err := filepath.WalkDir(dir, func(path string, e fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(dir, path) + top := strings.Split(rel, string(filepath.Separator))[0] + if e.IsDir() && path != dir && (top == "gml" || top == "cityjson" || strings.HasPrefix(e.Name(), ".")) { + return fs.SkipDir + } + if strings.HasPrefix(e.Name(), ".") { + return nil + } + if !e.IsDir() && rel != attribution.FileName && isModelFile(path) { + problems = append(problems, fmt.Errorf("%s lies outside gml/ and cityjson/ and would not be imported", path)) + } + return nil + }) + if err != nil { + problems = append(problems, fmt.Errorf("read %s: %w", dir, err)) + } + return problems +} + +// isModelFile reports whether path has a model file extension, gzipped or not. +func isModelFile(path string) bool { + return modelExtensions[strings.ToLower(filepath.Ext(strings.TrimSuffix(strings.ToLower(path), ".gz")))] +} + +// checkModelFolder checks every model file under one gml/ or cityjson/ folder. +func checkModelFolder(in importInput, srid int) []error { + var problems []error + err := filepath.WalkDir(in.path, func(path string, e fs.DirEntry, err error) error { + if err != nil || e.IsDir() || !isModelFile(path) { + return err + } + check := checkCityGMLFile + if in.format == "cityjson" { + check = checkCityJSONFile + } + if err := check(path, srid); err != nil { + problems = append(problems, fmt.Errorf("%s: %w", path, err)) + } + return nil + }) + if err != nil { + problems = append(problems, fmt.Errorf("read %s: %w", in.path, err)) + } + return problems +} + +// openModelFile opens path, decompressing a .gz file. +func openModelFile(path string) (io.ReadCloser, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + if !strings.HasSuffix(strings.ToLower(path), ".gz") { + return f, nil + } + gz, err := gzip.NewReader(f) + if err != nil { + f.Close() + return nil, fmt.Errorf("not a readable gzip file: %w", err) + } + return struct { + io.Reader + io.Closer + }{gz, f}, nil +} + +// checkCityGMLFile checks that a file is CityGML and that every srsName in its +// first gmlHeaderBytes resolves to srid. +func checkCityGMLFile(path string, srid int) error { + r, err := openModelFile(path) + if err != nil { + return err + } + defer r.Close() + head, err := io.ReadAll(io.LimitReader(r, gmlHeaderBytes)) + if err != nil { + return fmt.Errorf("cannot be read: %w", err) + } + if !strings.Contains(string(head), "CityModel") { + return errors.New("is not CityGML: no CityModel element at its start") + } + names := srsNamePattern.FindAllStringSubmatch(string(head), -1) + if len(names) == 0 { + return fmt.Errorf("declares no srsName in its first %d bytes", gmlHeaderBytes) + } + seen := map[string]bool{} + for _, m := range names { + if !seen[m[1]] { + seen[m[1]] = true + if err := checkCRS(m[1], srid); err != nil { + return err + } + } + } + return nil +} + +// checkCityJSONFile checks that a file is CityJSON and that its +// metadata.referenceSystem resolves to srid. The whole file is scanned, as a +// CityJSON writer may put metadata after CityObjects. +func checkCityJSONFile(path string, srid int) error { + r, err := openModelFile(path) + if err != nil { + return err + } + defer r.Close() + isCityJSON, referenceSystem, err := scanCityJSON(r) + if err != nil { + return fmt.Errorf("cannot be read: %w", err) + } + if !isCityJSON { + return errors.New(`is not CityJSON: no "type":"CityJSON"`) + } + if referenceSystem == "" { + return errors.New("declares no metadata.referenceSystem") + } + return checkCRS(referenceSystem, srid) +} + +// scanCityJSON streams r in chunks that overlap by more than a match, so a +// pattern split across two reads is still found. +func scanCityJSON(r io.Reader) (isCityJSON bool, referenceSystem string, err error) { + const chunk, overlap = 1 << 20, 4096 + br := bufio.NewReaderSize(r, chunk) + var tail []byte + buf := make([]byte, chunk) + for { + n, readErr := io.ReadFull(br, buf) + window := append(tail, buf[:n]...) + isCityJSON = isCityJSON || strings.Contains(string(window), `"type":"CityJSON`) || strings.Contains(string(window), `"type": "CityJSON`) + if m := referenceSystemPattern.FindSubmatch(window); m != nil && referenceSystem == "" { + referenceSystem = string(m[1]) + } + if errors.Is(readErr, io.EOF) || errors.Is(readErr, io.ErrUnexpectedEOF) { + return isCityJSON, referenceSystem, nil + } + if readErr != nil { + return false, "", readErr + } + tail = append([]byte(nil), window[max(0, len(window)-overlap):]...) + } +} + +// checkCRS refuses a declared CRS that does not resolve to srid. +func checkCRS(name string, srid int) error { + code, err := resolveCRS(name) + if err != nil { + return err + } + if code != srid { + return fmt.Errorf("declares CRS %q (EPSG:%d), but this country's data must be in EPSG:%d", name, code, srid) + } + return nil +} diff --git a/internal/importer/check_test.go b/internal/importer/check_test.go new file mode 100644 index 0000000..96180c1 --- /dev/null +++ b/internal/importer/check_test.go @@ -0,0 +1,166 @@ +package importer + +import ( + "bytes" + "compress/gzip" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/thd-spatial-ai/city2tabula/internal/config" +) + +func TestResolveCRS(t *testing.T) { + tests := []struct { + name string + want int + wantErr bool + }{ + {"urn:adv:crs:ETRS89_UTM32*DE_DHHN2016_NH", 25832, false}, // Bavaria, NRW + {"urn:ogc:def:crs,crs:EPSG::31256", 31256, false}, // Vienna + {"urn:ogc:def:crs:EPSG::25832", 25832, false}, + {"https://www.opengis.net/def/crs/EPSG/0/5514", 5514, false}, // Brno + {"https://www.opengis.net/def/crs/EPSG/0/7415", 28992, false}, // 3DBAG, compound + {"EPSG:28992", 28992, false}, + {"EPSG:74150", 74150, false}, + {"unknown", 0, true}, // Prague + {"urn:ogc:drf:crs:EPSG::3879", 0, true}, // Helsinki, malformed + {"", 0, true}, + } + for _, tc := range tests { + got, err := resolveCRS(tc.name) + if (err != nil) != tc.wantErr || got != tc.want { + t.Errorf("resolveCRS(%q) = %d, %v; want %d, error %v", tc.name, got, err, tc.want, tc.wantErr) + } + } +} + +func gmlTile(srsName string) string { + return `` + + `` +} + +func writeFile(t *testing.T, path string, content string, gzipped bool) { + t.Helper() + data := []byte(content) + if gzipped { + var b bytes.Buffer + w := gzip.NewWriter(&b) + if _, err := w.Write(data); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + data = b.Bytes() + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } +} + +func TestCheckDataset(t *testing.T) { + const adv = "urn:adv:crs:ETRS89_UTM32*DE_DHHN2016_NH" + tests := []struct { + name string + country string + setup func(t *testing.T, staged, published string) + wantErr []string + }{ + {"valid gzipped CityGML in a region subfolder", "germany", func(t *testing.T, s, _ string) { + writeFile(t, filepath.Join(s, "gml", "region-a", "tile.gml.gz"), gmlTile(adv), true) + }, nil}, + {"CityJSON with metadata after a long CityObjects", "netherlands", func(t *testing.T, s, _ string) { + body := `{"CityObjects":{"a":{"attributes":{"pad":"` + strings.Repeat("x", 1_500_000) + `"}}},` + + `"metadata":{"referenceSystem":"https://www.opengis.net/def/crs/EPSG/0/7415"},"type":"CityJSON"}` + writeFile(t, filepath.Join(s, "cityjson", "tile.city.json.gz"), body, true) + }, nil}, + {"CRS of another country", "germany", func(t *testing.T, s, _ string) { + writeFile(t, filepath.Join(s, "gml", "tile.gml"), gmlTile("urn:ogc:def:crs,crs:EPSG::31256"), false) + }, []string{"EPSG:31256", "EPSG:25832"}}, + {"CRS not a recognised name", "czechia", func(t *testing.T, s, _ string) { + writeFile(t, filepath.Join(s, "gml", "tile.gml"), gmlTile("unknown"), false) + }, []string{`"unknown"`}}, + {"no CRS declared", "germany", func(t *testing.T, s, _ string) { + writeFile(t, filepath.Join(s, "gml", "tile.gml"), ``, false) + }, []string{"declares no srsName"}}, + {"not CityGML", "germany", func(t *testing.T, s, _ string) { + writeFile(t, filepath.Join(s, "gml", "tile.gml"), ``, false) + }, []string{"is not CityGML"}}, + {"model file outside the model folders", "germany", func(t *testing.T, s, _ string) { + writeFile(t, filepath.Join(s, "gml", "tile.gml"), gmlTile(adv), false) + writeFile(t, filepath.Join(s, "extra", "tile.gml"), gmlTile(adv), false) + }, []string{"extra", "would not be imported"}}, + {"hidden folders are not read", "germany", func(t *testing.T, s, _ string) { + writeFile(t, filepath.Join(s, "gml", "tile.gml"), gmlTile(adv), false) + writeFile(t, filepath.Join(s, ".venv", "lib", "schema.json"), "{}", false) + }, nil}, + {"dataset_id used by a published folder with another attribution", "germany", func(t *testing.T, s, p string) { + writeFile(t, filepath.Join(s, "gml", "tile.gml"), gmlTile(adv), false) + other := writeDataset(t, p, "other", "de-test-lod2") + data, _ := os.ReadFile(filepath.Join(other, "attribution.json")) + writeFile(t, filepath.Join(other, "attribution.json"), strings.Replace(string(data), "Example Provider,", "Another Provider,", 1), false) + }, []string{`dataset_id "de-test-lod2" is already used`}}, + {"dataset_id of the published folder it extends", "germany", func(t *testing.T, s, p string) { + writeFile(t, filepath.Join(s, "gml", "tile.gml"), gmlTile(adv), false) + writeDataset(t, p, "test", "de-test-lod2") + }, nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + stagingRoot, published := t.TempDir(), t.TempDir() + staged := writeDataset(t, stagingRoot, "staged", "de-test-lod2") + tc.setup(t, staged, published) + cfg := discoverConfig(published, filepath.Join(published, "no-lod3")) + cfg.Country = tc.country + + problems := CheckDataset(cfg, staged) + var all []string + for _, p := range problems { + all = append(all, p.Error()) + } + joined := strings.Join(all, "\n") + if len(tc.wantErr) == 0 && len(problems) > 0 { + t.Fatalf("want no problems, got:\n%s", joined) + } + for _, want := range tc.wantErr { + if !strings.Contains(joined, want) { + t.Errorf("problems do not mention %q:\n%s", want, joined) + } + } + }) + } +} + +func TestCheckDataset_UsesConfiguredSRID(t *testing.T) { + staged := writeDataset(t, t.TempDir(), "staged", "de-test-lod2") + writeFile(t, filepath.Join(staged, "gml", "tile.gml"), gmlTile("urn:ogc:def:crs:EPSG::25833"), false) + cfg := discoverConfig(t.TempDir(), t.TempDir()) + cfg.Country = "germany" + if problems := CheckDataset(cfg, staged); len(problems) == 0 { + t.Fatal("EPSG:25833 passed against the German default EPSG:25832") + } + cfg.CityDB = &config.CityDB{SRID: "25833"} + if problems := CheckDataset(cfg, staged); len(problems) != 0 { + t.Errorf("with CITYDB_SRID 25833: %v", problems) + } +} + +func TestCheckDataset_MissingAttributionAndModelFolders(t *testing.T) { + staged := t.TempDir() + cfg := discoverConfig(t.TempDir(), t.TempDir()) + cfg.Country = "germany" + joined := "" + for _, p := range CheckDataset(cfg, staged) { + joined += p.Error() + "\n" + } + for _, want := range []string{"attribution", "neither a gml nor a cityjson"} { + if !strings.Contains(joined, want) { + t.Errorf("problems do not mention %q:\n%s", want, joined) + } + } +}