From 62783c9873f367a08bd36b32c9fd299ad593e530 Mon Sep 17 00:00:00 2001 From: Jay Ravani Date: Mon, 26 Oct 2026 22:18:16 +0100 Subject: [PATCH] feat: import only the files whose extent a bbox reaches A bbox import passed citydb-tool the whole dataset folder, which read and parsed every file and kept the features in the bbox. With a whole provider on disk, every region import cost a full read of the folder, and since the import was recorded against the fingerprint of the whole folder, adding any tile invalidated every region's record. A bbox import now reads each model file's declared extent first: a CityGML boundedBy envelope, searched in the first 16 KiB and then up to 1 MiB, or a CityJSON metadata.geographicalExtent with its referenceSystem. The bbox is transformed into each file's CRS once, and only the intersecting files are passed to citydb-tool, through an argument file. A file without a readable extent or a recognised CRS is always passed, so citydb-tool's own filter decides for it; a bbox no file reaches runs no import. The bbox is recorded against the fingerprint of the selected files alone, and a new record replaces only earlier ones of the same bbox and mode, so tiles added elsewhere leave a region's record valid. A whole-folder import of the current files still covers any bbox. On all 18,297 Bavarian tiles (142 GB) in one folder, a region of about 2 by 1 km selects 4 files and imports in 9 s; a repeat run takes about 1 s, and 3 s after another region was added. --- CHANGELOG.md | 13 + docs/deployment/adding-data.md | 12 + internal/importer/citydb.go | 18 +- internal/importer/citydb_test.go | 4 +- internal/importer/extent.go | 96 ++++++-- .../import_extent_integration_test.go | 92 ++++++- internal/importer/select.go | 224 ++++++++++++++++++ internal/importer/select_test.go | 39 +++ 8 files changed, 467 insertions(+), 31 deletions(-) create mode 100644 internal/importer/select.go create mode 100644 internal/importer/select_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f81b082..2e9f4dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,19 @@ Each release is listed with its breaking changes first. Releases before v0.8.0 are described in their tag messages. +## Unreleased + +Features: +- A bbox import (POST /api/v1/runs, -import-data -bbox) passes + citydb-tool only the files whose declared extent intersects the bbox: + a CityGML boundedBy envelope, or a CityJSON metadata + geographicalExtent. It records the bbox against those files alone, + so a region import reads only that region's tiles, and adding tiles + elsewhere in the dataset folder leaves earlier regions' records valid. + A file without a readable extent or a recognised CRS is always + passed. On all 18,297 Bavarian tiles in one folder, a region of about + 2 by 1 km imports 4 files in 9 s; a repeat run takes about 1 s. + ## v0.14.0 (2026-10-26) Features: diff --git a/docs/deployment/adding-data.md b/docs/deployment/adding-data.md index f309d09..db4242e 100644 --- a/docs/deployment/adding-data.md +++ b/docs/deployment/adding-data.md @@ -23,6 +23,18 @@ COUNTRY=germany c2t -import-data # afterwards, as often as needed: imports wha A folder counts as changed when the path, size or modification time of any file in its `gml/` or `cityjson/` folder changes. A run for a bbox (`POST /api/v1/runs`, or `-import-data -bbox`) after a whole-folder import of unchanged files does not run citydb-tool either. +## Importing one region + +A run for a bbox imports only the files whose declared extent intersects it: a CityGML file's `boundedBy` envelope, or a CityJSON file's `metadata.geographicalExtent` in its `referenceSystem`. A file without a readable extent, or with a CRS that is not recognised (see [Checking files before publishing](#checking-files-before-publishing)), is always passed to citydb-tool, whose own bbox filter then applies. The bbox is recorded against the selected files alone, so: + +| Since the last run for the same bbox | Result | +|---|---| +| No selected file added, replaced or removed | citydb-tool is not run | +| Files added or changed in another region | citydb-tool is not run | +| A selected file added, replaced or removed | The selected files are imported again; existing buildings are skipped | + +With all 18,297 Bavarian tiles (142 GB) in one dataset folder, a region of about 2 by 1 km selects 4 files, imports in 9 s, and a repeat run takes about 1 s. + !!! warning "One invalid attribution file stops the country's import" Before anything is imported, every dataset folder of the country is checked: it needs a valid `attribution.json` ([Dataset attribution file](../code/attribution/index.md)) and a `gml/` or `cityjson/` folder, and no model file may lie directly in `data/lod2//`. One failing folder stops the whole import for that country, with an error naming the folder; nothing is imported until it is fixed. A scheduled job should report the error. diff --git a/internal/importer/citydb.go b/internal/importer/citydb.go index fa04daa..d754ede 100644 --- a/internal/importer/citydb.go +++ b/internal/importer/citydb.go @@ -63,9 +63,23 @@ func testCityDBExecPath(cityDBExecPath string) error { // importCityDBFiles imports each of a dataset's model folders with its own // format into the dataset's schema, tagging every feature with the dataset_id // as its lineage. -func importCityDBFiles(cityDBExecPath string, d Dataset, config *config.Config, bbox, bboxMode string) error { +func importCityDBFiles(cityDBExecPath string, d Dataset, config *config.Config, bbox, bboxMode string, selection map[string][]string) error { for _, in := range d.Inputs { - cmd := getCityDBImportCommand(cityDBExecPath, in.path, d.Schema, in.format, d.Attribution.DatasetID, config, bbox, bboxMode) + dataPath := in.path + if selection != nil { + files := selection[in.path] + if len(files) == 0 { + utils.Info.Printf("%s %s: no file intersects the bbox, nothing to import", d.Attribution.DatasetID, in.label) + continue + } + argFile, err := writeArgFile(files) + if err != nil { + return fmt.Errorf("list the files of %s to import: %w", in.path, err) + } + defer os.Remove(argFile) + dataPath = "@" + argFile + } + cmd := getCityDBImportCommand(cityDBExecPath, dataPath, d.Schema, in.format, d.Attribution.DatasetID, config, bbox, bboxMode) if err := executeCityDBCommand(cmd, fmt.Sprintf("%s %s", d.Attribution.DatasetID, in.label)); err != nil { return err } diff --git a/internal/importer/citydb_test.go b/internal/importer/citydb_test.go index d667bff..7e16ecc 100644 --- a/internal/importer/citydb_test.go +++ b/internal/importer/citydb_test.go @@ -211,7 +211,7 @@ func TestImportCityDBFiles_BothFormatsAttemptedOnSuccess(t *testing.T) { {"citygml", "CityGML", filepath.Join(dataDir, "gml")}, {"cityjson", "CityJSON", filepath.Join(dataDir, "cityjson")}, }, Attribution: attribution.Attribution{DatasetID: "ds-1"}} - if err := importCityDBFiles(exe, d, cfg, "", ""); err != nil { + if err := importCityDBFiles(exe, d, cfg, "", "", nil); err != nil { t.Fatalf("importCityDBFiles: %v", err) } @@ -246,7 +246,7 @@ func TestImportCityDBFiles_CityGMLFailureShortCircuitsCityJSON(t *testing.T) { {"citygml", "CityGML", filepath.Join(dataDir, "gml")}, {"cityjson", "CityJSON", filepath.Join(dataDir, "cityjson")}, }, Attribution: attribution.Attribution{DatasetID: "ds-1"}} - if err := importCityDBFiles(exe, d, cfg, "", ""); err == nil { + if err := importCityDBFiles(exe, d, cfg, "", "", nil); err == nil { t.Fatal("expected an error when the citygml import fails, got nil") } diff --git a/internal/importer/extent.go b/internal/importer/extent.go index d2527f6..db8e0de 100644 --- a/internal/importer/extent.go +++ b/internal/importer/extent.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "io/fs" + "os" "path/filepath" "strconv" "strings" @@ -63,23 +64,31 @@ func parseImportBbox(s string, defaultSRID string) (importBbox, error) { // in the dataset's model folders, so adding, replacing or removing a file // changes it. func inputFingerprint(d Dataset) (string, error) { - h := sha256.New() + var files []string for _, in := range d.Inputs { err := filepath.WalkDir(in.path, func(path string, e fs.DirEntry, err error) error { - if err != nil || e.IsDir() { - return err - } - info, err := e.Info() - if err != nil { - return err + if err == nil && !e.IsDir() { + files = append(files, path) } - fmt.Fprintf(h, "%s\x00%d\x00%d\n", path, info.Size(), info.ModTime().UnixNano()) - return nil + return err }) if err != nil { return "", fmt.Errorf("fingerprint %s: %w", in.path, err) } } + return fingerprintFiles(files) +} + +// fingerprintFiles hashes the path, size and modification time of files. +func fingerprintFiles(files []string) (string, error) { + h := sha256.New() + for _, path := range files { + info, err := os.Stat(path) + if err != nil { + return "", fmt.Errorf("fingerprint %s: %w", path, err) + } + fmt.Fprintf(h, "%s\x00%d\x00%d\n", path, info.Size(), info.ModTime().UnixNano()) + } return hex.EncodeToString(h.Sum(nil)), nil } @@ -126,15 +135,18 @@ func extentImported(conn *pgxpool.Pool, d Dataset, fingerprint string, b importB return covered, nil } -// recordExtent stores b as imported for d, and drops rows recorded against -// source files that have since changed. +// recordExtent stores b as imported for d from the files fingerprint stands for, +// and drops the rows of the same bbox and mode recorded from other files. Rows of +// other bboxes stay: each is checked against its own files' fingerprint. func recordExtent(conn *pgxpool.Pool, d Dataset, fingerprint string, b importBbox, mode string) error { ctx, cancel := context.WithTimeout(context.Background(), extentTimeout) defer cancel() table := pgx.Identifier{d.Schema, extentTable}.Sanitize() err := pgx.BeginFunc(ctx, conn, func(tx pgx.Tx) error { - if _, err := tx.Exec(ctx, `DELETE FROM `+table+` WHERE dataset_id = $1 AND input_fingerprint <> $2`, - d.Attribution.DatasetID, fingerprint); err != nil { + if _, err := tx.Exec(ctx, `DELETE FROM `+table+` + WHERE dataset_id = $1 AND input_fingerprint <> $2 AND bbox_mode = $3 + AND ST_Equals(geom, ST_Transform(ST_MakeEnvelope($4, $5, $6, $7, $8::int), 4326))`, + d.Attribution.DatasetID, fingerprint, mode, b.xmin, b.ymin, b.xmax, b.ymax, b.srid); err != nil { return err } _, err := tx.Exec(ctx, `INSERT INTO `+table+` (dataset_id, input_fingerprint, bbox_mode, geom) @@ -159,31 +171,63 @@ var wholeFolder = importBbox{xmin: -180, ymin: -90, xmax: 180, ymax: 90, srid: 4 // import when conn is nil, as in tests that never reach a real database. func importDatasetOnce(conn *pgxpool.Pool, cityDBExecPath string, d Dataset, cfg *config.Config, bbox, bboxMode string) error { if cfg.CityDB.ImportLimit > 0 || conn == nil { - return importCityDBFiles(cityDBExecPath, d, cfg, bbox, bboxMode) - } - b, mode := wholeFolder, "intersects" - if bbox != "" { - parsed, err := parseImportBbox(bbox, cfg.CityDB.SRID) - if err != nil { - return err - } - b, mode = parsed, bboxMode + return importCityDBFiles(cityDBExecPath, d, cfg, bbox, bboxMode, nil) } fingerprint, err := inputFingerprint(d) if err != nil { return err } + if bbox == "" { + return importWholeFolderOnce(conn, cityDBExecPath, d, cfg, fingerprint) + } + b, err := parseImportBbox(bbox, cfg.CityDB.SRID) + if err != nil { + return err + } + return importBboxOnce(conn, cityDBExecPath, d, cfg, b, bbox, bboxMode, fingerprint) +} + +// importWholeFolderOnce imports the whole of d unless the same files were +// imported whole before. +func importWholeFolderOnce(conn *pgxpool.Pool, cityDBExecPath string, d Dataset, cfg *config.Config, fingerprint string) error { + covered, err := extentImported(conn, d, fingerprint, wholeFolder, "intersects") + if err != nil || covered { + if covered { + utils.Info.Printf("%s already imported whole from unchanged files, skipping citydb-tool", d.Attribution.DatasetID) + } + return err + } + if err := importCityDBFiles(cityDBExecPath, d, cfg, "", "", nil); err != nil { + return err + } + return recordExtent(conn, d, fingerprint, wholeFolder, "intersects") +} - covered, err := extentImported(conn, d, fingerprint, b, mode) +// importBboxOnce imports the files of d whose declared extent intersects b, and +// records b against those files alone, so files added elsewhere leave the record +// valid. A whole-folder import of the current files also covers b. +func importBboxOnce(conn *pgxpool.Pool, cityDBExecPath string, d Dataset, cfg *config.Config, b importBbox, bbox, mode, folderFingerprint string) error { + covered, err := extentImported(conn, d, folderFingerprint, b, mode) if err != nil { return err } + ctx, cancel := context.WithTimeout(context.Background(), extentTimeout) + defer cancel() + var selection map[string][]string + var fingerprint string + if !covered { + if selection, fingerprint, err = selectFiles(ctx, conn, d, b); err != nil { + return err + } + if covered, err = extentImported(conn, d, fingerprint, b, mode); err != nil { + return err + } + } if covered { - utils.Info.Printf("%s already imported from unchanged files for bbox %q, skipping citydb-tool", d.Attribution.DatasetID, bbox) + utils.Info.Printf("%s already imported from unchanged files for bbox %s, skipping citydb-tool", d.Attribution.DatasetID, bbox) return nil } - - if err := importCityDBFiles(cityDBExecPath, d, cfg, bbox, bboxMode); err != nil { + if err := importCityDBFiles(cityDBExecPath, d, cfg, bbox, mode, selection); err != nil { return err } return recordExtent(conn, d, fingerprint, b, mode) diff --git a/internal/importer/import_extent_integration_test.go b/internal/importer/import_extent_integration_test.go index 6d27106..b14c588 100644 --- a/internal/importer/import_extent_integration_test.go +++ b/internal/importer/import_extent_integration_test.go @@ -26,7 +26,9 @@ func extentConfig(t *testing.T, ctx context.Context, schema string) (*config.Con toolDir := t.TempDir() logPath := filepath.Join(toolDir, "calls.txt") - script := fmt.Sprintf("#!/bin/sh\necho \"$@\" >> %q\nexit 0\n", logPath) + // An argument file (@path) is logged as a FILES line, since the importer + // removes it once citydb-tool returns. + script := fmt.Sprintf("#!/bin/sh\nfor a in \"$@\"; do case \"$a\" in @*) printf 'FILES %%s\\n' \"$(tr '\\n' ' ' < \"${a#@}\")\" >> %[1]q;; esac; done\necho \"$@\" >> %[1]q\nexit 0\n", logPath) if err := os.WriteFile(filepath.Join(toolDir, "citydb"), []byte(script), 0o755); err != nil { t.Fatalf("write fake citydb: %v", err) } @@ -150,3 +152,91 @@ func TestImportCityDBData_SkipsUnchangedWholeFolder(t *testing.T) { } } } + +// tileGML is a CityGML file whose envelope covers x0..x0+2000, y0..y0+2000 in +// EPSG:25832. +func tileGML(x0, y0 float64) string { + return fmt.Sprintf(``+ + `%[1]f %[2]f 0%[3]f %[4]f 100`+ + ``, x0, y0, x0+2000, y0+2000) +} + +// bboxIn4326 returns an import bbox for an EPSG:25832 rectangle. +func bboxIn4326(t *testing.T, ctx context.Context, xmin, ymin, xmax, ymax float64) string { + t.Helper() + var b [4]float64 + if err := testPool.QueryRow(ctx, ` + SELECT ST_XMin(e), ST_YMin(e), ST_XMax(e), ST_YMax(e) + FROM (SELECT ST_Transform(ST_MakeEnvelope($1, $2, $3, $4, 25832), 4326) AS e) s`, + xmin, ymin, xmax, ymax).Scan(&b[0], &b[1], &b[2], &b[3]); err != nil { + t.Fatalf("transform bbox: %v", err) + } + return fmt.Sprintf("%f,%f,%f,%f,4326", b[0], b[1], b[2], b[3]) +} + +// lastFiles returns the file names of the last argument file citydb was given. +func lastFiles(t *testing.T, logPath string) string { + t.Helper() + data, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("read citydb call log %s: %v", logPath, err) + } + last := "" + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "FILES ") { + last = "" + for _, f := range strings.Fields(strings.TrimPrefix(line, "FILES ")) { + last += filepath.Base(strings.Trim(f, `"`)) + " " + } + } + } + return strings.TrimSpace(last) +} + +// TestImportCityDBData_SelectsFilesInTheBbox pins that a bbox import passes +// citydb-tool only the files whose declared extent intersects the bbox, and +// records the bbox against those files alone: adding a file elsewhere leaves the +// record valid, and a bbox no file reaches runs no import. +func TestImportCityDBData_SelectsFilesInTheBbox(t *testing.T) { + ctx := context.Background() + cfg, logPath, gml := extentConfig(t, ctx, "extent_select_lod2") + if err := os.Remove(filepath.Join(gml, "tile-1.gml")); err != nil { + t.Fatal(err) + } + write := func(name string, x0, y0 float64) { + if err := os.WriteFile(filepath.Join(gml, name), []byte(tileGML(x0, y0)), 0o644); err != nil { + t.Fatal(err) + } + } + write("a.gml", 786000, 5416000) + write("b.gml", 790000, 5416000) + regionA := bboxIn4326(t, ctx, 786500, 5416500, 787500, 5417500) + regionB := bboxIn4326(t, ctx, 790500, 5416500, 793500, 5417500) + elsewhere := bboxIn4326(t, ctx, 800500, 5416500, 801500, 5417500) + + steps := []struct { + name, bbox, add string + wantCalls int + wantFiles string + }{ + {name: "region A", bbox: regionA, wantCalls: 1, wantFiles: "a.gml"}, + {name: "region A again", bbox: regionA, wantCalls: 1, wantFiles: "a.gml"}, + {name: "region A after a tile is added in region B", bbox: regionA, add: "c.gml", wantCalls: 1, wantFiles: "a.gml"}, + {name: "region B", bbox: regionB, wantCalls: 2, wantFiles: "b.gml c.gml"}, + {name: "a bbox no file reaches", bbox: elsewhere, wantCalls: 2, wantFiles: "b.gml c.gml"}, + } + for _, s := range steps { + if s.add != "" { + write(s.add, 792000, 5416000) + } + if err := importer.ImportCityDBData(testPool, cfg, s.bbox, "intersects"); err != nil { + t.Fatalf("%s: ImportCityDBData: %v", s.name, err) + } + if got := importCalls(t, logPath); got != s.wantCalls { + t.Fatalf("%s: citydb import called %d times in total, want %d", s.name, got, s.wantCalls) + } + if got := lastFiles(t, logPath); got != s.wantFiles { + t.Fatalf("%s: last import got files %q, want %q", s.name, got, s.wantFiles) + } + } +} diff --git a/internal/importer/select.go b/internal/importer/select.go new file mode 100644 index 0000000..5d05d60 --- /dev/null +++ b/internal/importer/select.go @@ -0,0 +1,224 @@ +package importer + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/thd-spatial-ai/city2tabula/internal/utils" +) + +// fileExtent is the extent a model file declares, in the CRS it names. +type fileExtent struct { + minX, minY, maxX, maxY float64 + crs string +} + +var ( + // gmlEnvelope matches a CityGML boundedBy envelope; its corners may carry + // attributes such as srsDimension. + gmlEnvelope = regexp.MustCompile(`(?s)]*\bsrsName="([^"]*)"[^>]*>\s*` + + `]*>([^<]*)\s*]*>([^<]*)`) + cityJSONMetadata = regexp.MustCompile(`"metadata"\s*:\s*\{`) + geographicalExtent = regexp.MustCompile(`"geographicalExtent"\s*:\s*\[([^\]]*)\]`) +) + +// envelopeFirstRead is how much of a CityGML file is read before the rest of +// gmlHeaderBytes. +const envelopeFirstRead = 16 << 10 + +// metadataBytes is how much of a CityJSON file after the "metadata" key is +// searched for the dataset's geographicalExtent and referenceSystem. +const metadataBytes = 16 << 10 + +// readFileExtent returns the extent a model file declares: a CityGML file's +// boundedBy envelope in its first gmlHeaderBytes, or a CityJSON file's +// metadata.geographicalExtent and referenceSystem. ok is false when the file +// declares none, or none that can be read. +func readFileExtent(path, format string) (fileExtent, bool, error) { + r, err := openModelFile(path) + if err != nil { + return fileExtent{}, false, err + } + defer r.Close() + if format == "cityjson" { + return cityJSONExtent(r) + } + // Most providers' envelope ends within the first few KiB, so the rest of + // gmlHeaderBytes is read only when it is not found there. + head, err := io.ReadAll(io.LimitReader(r, envelopeFirstRead)) + if err != nil { + return fileExtent{}, false, err + } + m := gmlEnvelope.FindSubmatch(head) + if m == nil && len(head) == envelopeFirstRead { + rest, err := io.ReadAll(io.LimitReader(r, gmlHeaderBytes-envelopeFirstRead)) + if err != nil { + return fileExtent{}, false, err + } + m = gmlEnvelope.FindSubmatch(append(head, rest...)) + } + if m == nil { + return fileExtent{}, false, nil + } + lower, upper := strings.Fields(string(m[2])), strings.Fields(string(m[3])) + if len(lower) < 2 || len(upper) < 2 { + return fileExtent{}, false, nil + } + return parseExtent(string(m[1]), lower[0], lower[1], upper[0], upper[1]) +} + +// cityJSONExtent streams r to its "metadata" object, which CityJSON writers put +// before or after CityObjects; a CityObject's own geographicalExtent comes +// earlier in some files, so only the one inside metadata counts. +func cityJSONExtent(r io.Reader) (fileExtent, bool, error) { + br := bufio.NewReaderSize(r, 1<<20) + var window []byte + buf := make([]byte, 1<<20) + for { + n, readErr := io.ReadFull(br, buf) + window = append(window, buf[:n]...) + if loc := cityJSONMetadata.FindIndex(window); loc != nil { + for len(window)-loc[0] < metadataBytes && readErr == nil { + n, readErr = io.ReadFull(br, buf) + window = append(window, buf[:n]...) + } + return extentFromMetadata(window[loc[0]:min(len(window), loc[0]+metadataBytes)]) + } + if errors.Is(readErr, io.EOF) || errors.Is(readErr, io.ErrUnexpectedEOF) { + return fileExtent{}, false, nil + } + if readErr != nil { + return fileExtent{}, false, readErr + } + window = window[max(0, len(window)-4096):] + } +} + +func extentFromMetadata(metadata []byte) (fileExtent, bool, error) { + e := geographicalExtent.FindSubmatch(metadata) + crs := referenceSystemPattern.FindSubmatch(metadata) + if e == nil || crs == nil { + return fileExtent{}, false, nil + } + v := strings.Split(string(e[1]), ",") + if len(v) != 6 { // minx, miny, minz, maxx, maxy, maxz + return fileExtent{}, false, nil + } + return parseExtent(string(crs[1]), v[0], v[1], v[3], v[4]) +} + +func parseExtent(crs string, values ...string) (fileExtent, bool, error) { + var f [4]float64 + for i, s := range values { + v, err := strconv.ParseFloat(strings.TrimSpace(s), 64) + if err != nil { + return fileExtent{}, false, nil + } + f[i] = v + } + return fileExtent{minX: f[0], minY: f[1], maxX: f[2], maxY: f[3], crs: crs}, true, nil +} + +// selectFiles returns, per model folder of d, the files whose declared extent +// intersects b, and their fingerprint. A file whose extent or CRS cannot be +// read is selected, so citydb-tool's own bbox filter decides for it. +func selectFiles(ctx context.Context, conn *pgxpool.Pool, d Dataset, b importBbox) (map[string][]string, string, error) { + inEPSG := map[int]importBbox{} + selection := map[string][]string{} + var all []string + total := 0 + for _, in := range d.Inputs { + files, err := modelFiles(in.path) + if err != nil { + return nil, "", err + } + total += len(files) + for _, path := range files { + keep, err := intersects(ctx, conn, path, in.format, b, inEPSG) + if err != nil { + return nil, "", err + } + if keep { + selection[in.path] = append(selection[in.path], path) + all = append(all, path) + } + } + } + utils.Info.Printf("%s: %d of %d files intersect the bbox", d.Attribution.DatasetID, len(all), total) + fingerprint, err := fingerprintFiles(all) + return selection, fingerprint, err +} + +// intersects reports whether path's declared extent intersects b, transforming +// b once per CRS into inEPSG. +func intersects(ctx context.Context, conn *pgxpool.Pool, path, format string, b importBbox, inEPSG map[int]importBbox) (bool, error) { + e, ok, err := readFileExtent(path, format) + if err != nil { + return false, fmt.Errorf("read extent of %s: %w", path, err) + } + if !ok { + return true, nil + } + epsg, err := resolveCRS(e.crs) + if err != nil { + return true, nil + } + t, done := inEPSG[epsg] + if !done { + if err := conn.QueryRow(ctx, ` + SELECT ST_XMin(e), ST_YMin(e), ST_XMax(e), ST_YMax(e) + FROM (SELECT ST_Transform(ST_MakeEnvelope($1, $2, $3, $4, $5::int), $6::int) AS e) s`, + b.xmin, b.ymin, b.xmax, b.ymax, b.srid, epsg, + ).Scan(&t.xmin, &t.ymin, &t.xmax, &t.ymax); err != nil { + return false, fmt.Errorf("transform bbox to EPSG:%d for %s: %w", epsg, path, err) + } + inEPSG[epsg] = t + } + return e.minX <= t.xmax && e.maxX >= t.xmin && e.minY <= t.ymax && e.maxY >= t.ymin, nil +} + +// modelFiles lists the model files under one model folder, in path order. +func modelFiles(dir string) ([]string, error) { + var files []string + err := filepath.WalkDir(dir, func(path string, e fs.DirEntry, err error) error { + if err != nil || e.IsDir() || !isModelFile(path) { + return err + } + files = append(files, path) + return nil + }) + if err != nil { + return nil, fmt.Errorf("list %s: %w", dir, err) + } + sort.Strings(files) + return files, nil +} + +// writeArgFile writes paths to a citydb-tool argument file, one quoted path per +// line, and returns its name. +func writeArgFile(paths []string) (string, error) { + f, err := os.CreateTemp("", "c2t-import-*.args") + if err != nil { + return "", err + } + defer f.Close() + for _, p := range paths { + quoted := `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(p) + `"` + if _, err := fmt.Fprintln(f, quoted); err != nil { + return "", err + } + } + return f.Name(), nil +} diff --git a/internal/importer/select_test.go b/internal/importer/select_test.go new file mode 100644 index 0000000..ce09b02 --- /dev/null +++ b/internal/importer/select_test.go @@ -0,0 +1,39 @@ +package importer + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestReadFileExtent(t *testing.T) { + dir := t.TempDir() + envelope := `` + + `786000.0 5416000.0 0.0` + + `788000.0 5418000.0 400.0` + cityJSON := `{"type":"CityJSON","CityObjects":{"b":{"geographicalExtent":[1,2,3,4,5,6],"attributes":{"pad":"` + + strings.Repeat("x", 1_200_000) + `"}}},"metadata":{"geographicalExtent":[197420.5,457109.6,61.1,197431.0,457120.3,68.5],` + + `"referenceSystem":"https://www.opengis.net/def/crs/EPSG/0/7415"}}` + tests := []struct { + name, file, content, format string + want fileExtent + wantOK bool + }{ + {"CityGML envelope, corners with attributes, gzipped", "a.gml.gz", envelope, "citygml", + fileExtent{786000, 5416000, 788000, 5418000, "urn:adv:crs:ETRS89_UTM32*DE_DHHN2016_NH"}, true}, + {"CityJSON extent from metadata, not from a CityObject", "b.city.json.gz", cityJSON, "cityjson", + fileExtent{197420.5, 457109.6, 197431.0, 457120.3, "https://www.opengis.net/def/crs/EPSG/0/7415"}, true}, + {"CityGML without an envelope", "c.gml", ``, "citygml", fileExtent{}, false}, + {"CityJSON metadata without an extent", "d.city.json", `{"type":"CityJSON","metadata":{"referenceSystem":"EPSG:5514"},"CityObjects":{}}`, "cityjson", fileExtent{}, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(dir, tc.file) + writeFile(t, path, tc.content, strings.HasSuffix(tc.file, ".gz")) + got, ok, err := readFileExtent(path, tc.format) + if err != nil || ok != tc.wantOK || got != tc.want { + t.Errorf("readFileExtent = %+v, %v, %v; want %+v, %v", got, ok, err, tc.want, tc.wantOK) + } + }) + } +}