feat: import only the files whose extent a bbox reaches

CHANGELOG.md

@@ -3,6 +3,19 @@
 Each release is listed with its breaking changes first. Releases before
 v0.8.0 are described in their tag messages.
 
+## Unreleased
+
+Features:
+- A bbox import (POST /api/v1/runs, -import-data -bbox) passes
+  citydb-tool only the files whose declared extent intersects the bbox:
+  a CityGML boundedBy envelope, or a CityJSON metadata
+  geographicalExtent. It records the bbox against those files alone,
+  so a region import reads only that region's tiles, and adding tiles
+  elsewhere in the dataset folder leaves earlier regions' records valid.
+  A file without a readable extent or a recognised CRS is always
+  passed. On all 18,297 Bavarian tiles in one folder, a region of about
+  2 by 1 km imports 4 files in 9 s; a repeat run takes about 1 s.
+
 ## v0.14.0 (2026-10-26)
 
 Features:

docs/deployment/adding-data.md

@@ -23,6 +23,18 @@ COUNTRY=germany c2t -import-data   # afterwards, as often as needed: imports wha
 
 A folder counts as changed when the path, size or modification time of any file in its `gml/` or `cityjson/` folder changes. A run for a bbox (`POST /api/v1/runs`, or `-import-data -bbox`) after a whole-folder import of unchanged files does not run citydb-tool either.
 
+## Importing one region
+
+A run for a bbox imports only the files whose declared extent intersects it: a CityGML file's `boundedBy` envelope, or a CityJSON file's `metadata.geographicalExtent` in its `referenceSystem`. A file without a readable extent, or with a CRS that is not recognised (see [Checking files before publishing](#checking-files-before-publishing)), is always passed to citydb-tool, whose own bbox filter then applies. The bbox is recorded against the selected files alone, so:
+
+| Since the last run for the same bbox | Result |
+|---|---|
+| No selected file added, replaced or removed | citydb-tool is not run |
+| Files added or changed in another region | citydb-tool is not run |
+| A selected file added, replaced or removed | The selected files are imported again; existing buildings are skipped |
+
+With all 18,297 Bavarian tiles (142 GB) in one dataset folder, a region of about 2 by 1 km selects 4 files, imports in 9 s, and a repeat run takes about 1 s.
+
 !!! warning "One invalid attribution file stops the country's import"
     Before anything is imported, every dataset folder of the country is checked: it needs a valid `attribution.json` ([Dataset attribution file](../code/attribution/index.md)) and a `gml/` or `cityjson/` folder, and no model file may lie directly in `data/lod2/<country>/`. One failing folder stops the whole import for that country, with an error naming the folder; nothing is imported until it is fixed. A scheduled job should report the error.
 

internal/importer/citydb.go

@@ -63,9 +63,23 @@ func testCityDBExecPath(cityDBExecPath string) error {
 // importCityDBFiles imports each of a dataset's model folders with its own
 // format into the dataset's schema, tagging every feature with the dataset_id
 // as its lineage.
-func importCityDBFiles(cityDBExecPath string, d Dataset, config *config.Config, bbox, bboxMode string) error {
+func importCityDBFiles(cityDBExecPath string, d Dataset, config *config.Config, bbox, bboxMode string, selection map[string][]string) error {
 	for _, in := range d.Inputs {
-		cmd := getCityDBImportCommand(cityDBExecPath, in.path, d.Schema, in.format, d.Attribution.DatasetID, config, bbox, bboxMode)
+		dataPath := in.path
+		if selection != nil {
+			files := selection[in.path]
+			if len(files) == 0 {
+				utils.Info.Printf("%s %s: no file intersects the bbox, nothing to import", d.Attribution.DatasetID, in.label)
+				continue
+			}
+			argFile, err := writeArgFile(files)
+			if err != nil {
+				return fmt.Errorf("list the files of %s to import: %w", in.path, err)
+			}
+			defer os.Remove(argFile)
+			dataPath = "@" + argFile
+		}
+		cmd := getCityDBImportCommand(cityDBExecPath, dataPath, d.Schema, in.format, d.Attribution.DatasetID, config, bbox, bboxMode)
 		if err := executeCityDBCommand(cmd, fmt.Sprintf("%s %s", d.Attribution.DatasetID, in.label)); err != nil {
 			return err
 		}

internal/importer/citydb_test.go

@@ -211,7 +211,7 @@ func TestImportCityDBFiles_BothFormatsAttemptedOnSuccess(t *testing.T) {
 		{"citygml", "CityGML", filepath.Join(dataDir, "gml")},
 		{"cityjson", "CityJSON", filepath.Join(dataDir, "cityjson")},
 	}, Attribution: attribution.Attribution{DatasetID: "ds-1"}}
-	if err := importCityDBFiles(exe, d, cfg, "", ""); err != nil {
+	if err := importCityDBFiles(exe, d, cfg, "", "", nil); err != nil {
 		t.Fatalf("importCityDBFiles: %v", err)
 	}
 
@@ -246,7 +246,7 @@ func TestImportCityDBFiles_CityGMLFailureShortCircuitsCityJSON(t *testing.T) {
 		{"citygml", "CityGML", filepath.Join(dataDir, "gml")},
 		{"cityjson", "CityJSON", filepath.Join(dataDir, "cityjson")},
 	}, Attribution: attribution.Attribution{DatasetID: "ds-1"}}
-	if err := importCityDBFiles(exe, d, cfg, "", ""); err == nil {
+	if err := importCityDBFiles(exe, d, cfg, "", "", nil); err == nil {
 		t.Fatal("expected an error when the citygml import fails, got nil")
 	}
 

internal/importer/extent.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io/fs"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -63,23 +64,31 @@ func parseImportBbox(s string, defaultSRID string) (importBbox, error) {
 // in the dataset's model folders, so adding, replacing or removing a file
 // changes it.
 func inputFingerprint(d Dataset) (string, error) {
-	h := sha256.New()
+	var files []string
 	for _, in := range d.Inputs {
 		err := filepath.WalkDir(in.path, func(path string, e fs.DirEntry, err error) error {
-			if err != nil || e.IsDir() {
-				return err
-			}
-			info, err := e.Info()
-			if err != nil {
-				return err
+			if err == nil && !e.IsDir() {
+				files = append(files, path)
 			}
-			fmt.Fprintf(h, "%s\x00%d\x00%d\n", path, info.Size(), info.ModTime().UnixNano())
-			return nil
+			return err
 		})
 		if err != nil {
 			return "", fmt.Errorf("fingerprint %s: %w", in.path, err)
 		}
 	}
+	return fingerprintFiles(files)
+}
+
+// fingerprintFiles hashes the path, size and modification time of files.
+func fingerprintFiles(files []string) (string, error) {
+	h := sha256.New()
+	for _, path := range files {
+		info, err := os.Stat(path)
+		if err != nil {
+			return "", fmt.Errorf("fingerprint %s: %w", path, err)
+		}
+		fmt.Fprintf(h, "%s\x00%d\x00%d\n", path, info.Size(), info.ModTime().UnixNano())
+	}
 	return hex.EncodeToString(h.Sum(nil)), nil
 }
 
@@ -126,15 +135,18 @@ func extentImported(conn *pgxpool.Pool, d Dataset, fingerprint string, b importB
 	return covered, nil
 }
 
-// recordExtent stores b as imported for d, and drops rows recorded against
-// source files that have since changed.
+// recordExtent stores b as imported for d from the files fingerprint stands for,
+// and drops the rows of the same bbox and mode recorded from other files. Rows of
+// other bboxes stay: each is checked against its own files' fingerprint.
 func recordExtent(conn *pgxpool.Pool, d Dataset, fingerprint string, b importBbox, mode string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), extentTimeout)
 	defer cancel()
 	table := pgx.Identifier{d.Schema, extentTable}.Sanitize()
 	err := pgx.BeginFunc(ctx, conn, func(tx pgx.Tx) error {
-		if _, err := tx.Exec(ctx, `DELETE FROM `+table+` WHERE dataset_id = $1 AND input_fingerprint <> $2`,
-			d.Attribution.DatasetID, fingerprint); err != nil {
+		if _, err := tx.Exec(ctx, `DELETE FROM `+table+`
+			WHERE dataset_id = $1 AND input_fingerprint <> $2 AND bbox_mode = $3
+			  AND ST_Equals(geom, ST_Transform(ST_MakeEnvelope($4, $5, $6, $7, $8::int), 4326))`,
+			d.Attribution.DatasetID, fingerprint, mode, b.xmin, b.ymin, b.xmax, b.ymax, b.srid); err != nil {
 			return err
 		}
 		_, err := tx.Exec(ctx, `INSERT INTO `+table+` (dataset_id, input_fingerprint, bbox_mode, geom)
@@ -159,31 +171,63 @@ var wholeFolder = importBbox{xmin: -180, ymin: -90, xmax: 180, ymax: 90, srid: 4
 // import when conn is nil, as in tests that never reach a real database.
 func importDatasetOnce(conn *pgxpool.Pool, cityDBExecPath string, d Dataset, cfg *config.Config, bbox, bboxMode string) error {
 	if cfg.CityDB.ImportLimit > 0 || conn == nil {
-		return importCityDBFiles(cityDBExecPath, d, cfg, bbox, bboxMode)
-	}
-	b, mode := wholeFolder, "intersects"
-	if bbox != "" {
-		parsed, err := parseImportBbox(bbox, cfg.CityDB.SRID)
-		if err != nil {
-			return err
-		}
-		b, mode = parsed, bboxMode
+		return importCityDBFiles(cityDBExecPath, d, cfg, bbox, bboxMode, nil)
 	}
 	fingerprint, err := inputFingerprint(d)
 	if err != nil {
 		return err
 	}
+	if bbox == "" {
+		return importWholeFolderOnce(conn, cityDBExecPath, d, cfg, fingerprint)
+	}
+	b, err := parseImportBbox(bbox, cfg.CityDB.SRID)
+	if err != nil {
+		return err
+	}
+	return importBboxOnce(conn, cityDBExecPath, d, cfg, b, bbox, bboxMode, fingerprint)
+}
+
+// importWholeFolderOnce imports the whole of d unless the same files were
+// imported whole before.
+func importWholeFolderOnce(conn *pgxpool.Pool, cityDBExecPath string, d Dataset, cfg *config.Config, fingerprint string) error {
+	covered, err := extentImported(conn, d, fingerprint, wholeFolder, "intersects")
+	if err != nil || covered {
+		if covered {
+			utils.Info.Printf("%s already imported whole from unchanged files, skipping citydb-tool", d.Attribution.DatasetID)
+		}
+		return err
+	}
+	if err := importCityDBFiles(cityDBExecPath, d, cfg, "", "", nil); err != nil {
+		return err
+	}
+	return recordExtent(conn, d, fingerprint, wholeFolder, "intersects")
+}
 
-	covered, err := extentImported(conn, d, fingerprint, b, mode)
+// importBboxOnce imports the files of d whose declared extent intersects b, and
+// records b against those files alone, so files added elsewhere leave the record
+// valid. A whole-folder import of the current files also covers b.
+func importBboxOnce(conn *pgxpool.Pool, cityDBExecPath string, d Dataset, cfg *config.Config, b importBbox, bbox, mode, folderFingerprint string) error {
+	covered, err := extentImported(conn, d, folderFingerprint, b, mode)
 	if err != nil {
 		return err
 	}
+	ctx, cancel := context.WithTimeout(context.Background(), extentTimeout)
+	defer cancel()
+	var selection map[string][]string
+	var fingerprint string
+	if !covered {
+		if selection, fingerprint, err = selectFiles(ctx, conn, d, b); err != nil {
+			return err
+		}
+		if covered, err = extentImported(conn, d, fingerprint, b, mode); err != nil {
+			return err
+		}
+	}
 	if covered {
-		utils.Info.Printf("%s already imported from unchanged files for bbox %q, skipping citydb-tool", d.Attribution.DatasetID, bbox)
+		utils.Info.Printf("%s already imported from unchanged files for bbox %s, skipping citydb-tool", d.Attribution.DatasetID, bbox)
 		return nil
 	}
-
-	if err := importCityDBFiles(cityDBExecPath, d, cfg, bbox, bboxMode); err != nil {
+	if err := importCityDBFiles(cityDBExecPath, d, cfg, bbox, mode, selection); err != nil {
 		return err
 	}
 	return recordExtent(conn, d, fingerprint, b, mode)

internal/importer/import_extent_integration_test.go

@@ -26,7 +26,9 @@ func extentConfig(t *testing.T, ctx context.Context, schema string) (*config.Con
 
 	toolDir := t.TempDir()
 	logPath := filepath.Join(toolDir, "calls.txt")
-	script := fmt.Sprintf("#!/bin/sh\necho \"$@\" >> %q\nexit 0\n", logPath)
+	// An argument file (@path) is logged as a FILES line, since the importer
+	// removes it once citydb-tool returns.
+	script := fmt.Sprintf("#!/bin/sh\nfor a in \"$@\"; do case \"$a\" in @*) printf 'FILES %%s\\n' \"$(tr '\\n' ' ' < \"${a#@}\")\" >> %[1]q;; esac; done\necho \"$@\" >> %[1]q\nexit 0\n", logPath)
 	if err := os.WriteFile(filepath.Join(toolDir, "citydb"), []byte(script), 0o755); err != nil {
 		t.Fatalf("write fake citydb: %v", err)
 	}
@@ -150,3 +152,91 @@ func TestImportCityDBData_SkipsUnchangedWholeFolder(t *testing.T) {
 		}
 	}
 }
+
+// tileGML is a CityGML file whose envelope covers x0..x0+2000, y0..y0+2000 in
+// EPSG:25832.
+func tileGML(x0, y0 float64) string {
+	return fmt.Sprintf(`<core:CityModel><gml:boundedBy><gml:Envelope srsName="urn:ogc:def:crs:EPSG::25832">`+
+		`<gml:lowerCorner>%[1]f %[2]f 0</gml:lowerCorner><gml:upperCorner>%[3]f %[4]f 100</gml:upperCorner>`+
+		`</gml:Envelope></gml:boundedBy></core:CityModel>`, x0, y0, x0+2000, y0+2000)
+}
+
+// bboxIn4326 returns an import bbox for an EPSG:25832 rectangle.
+func bboxIn4326(t *testing.T, ctx context.Context, xmin, ymin, xmax, ymax float64) string {
+	t.Helper()
+	var b [4]float64
+	if err := testPool.QueryRow(ctx, `
+		SELECT ST_XMin(e), ST_YMin(e), ST_XMax(e), ST_YMax(e)
+		FROM (SELECT ST_Transform(ST_MakeEnvelope($1, $2, $3, $4, 25832), 4326) AS e) s`,
+		xmin, ymin, xmax, ymax).Scan(&b[0], &b[1], &b[2], &b[3]); err != nil {
+		t.Fatalf("transform bbox: %v", err)
+	}
+	return fmt.Sprintf("%f,%f,%f,%f,4326", b[0], b[1], b[2], b[3])
+}
+
+// lastFiles returns the file names of the last argument file citydb was given.
+func lastFiles(t *testing.T, logPath string) string {
+	t.Helper()
+	data, err := os.ReadFile(logPath)
+	if err != nil {
+		t.Fatalf("read citydb call log %s: %v", logPath, err)
+	}
+	last := ""
+	for _, line := range strings.Split(string(data), "\n") {
+		if strings.HasPrefix(line, "FILES ") {
+			last = ""
+			for _, f := range strings.Fields(strings.TrimPrefix(line, "FILES ")) {
+				last += filepath.Base(strings.Trim(f, `"`)) + " "
+			}
+		}
+	}
+	return strings.TrimSpace(last)
+}
+
+// TestImportCityDBData_SelectsFilesInTheBbox pins that a bbox import passes
+// citydb-tool only the files whose declared extent intersects the bbox, and
+// records the bbox against those files alone: adding a file elsewhere leaves the
+// record valid, and a bbox no file reaches runs no import.
+func TestImportCityDBData_SelectsFilesInTheBbox(t *testing.T) {
+	ctx := context.Background()
+	cfg, logPath, gml := extentConfig(t, ctx, "extent_select_lod2")
+	if err := os.Remove(filepath.Join(gml, "tile-1.gml")); err != nil {
+		t.Fatal(err)
+	}
+	write := func(name string, x0, y0 float64) {
+		if err := os.WriteFile(filepath.Join(gml, name), []byte(tileGML(x0, y0)), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write("a.gml", 786000, 5416000)
+	write("b.gml", 790000, 5416000)
+	regionA := bboxIn4326(t, ctx, 786500, 5416500, 787500, 5417500)
+	regionB := bboxIn4326(t, ctx, 790500, 5416500, 793500, 5417500)
+	elsewhere := bboxIn4326(t, ctx, 800500, 5416500, 801500, 5417500)
+
+	steps := []struct {
+		name, bbox, add string
+		wantCalls       int
+		wantFiles       string
+	}{
+		{name: "region A", bbox: regionA, wantCalls: 1, wantFiles: "a.gml"},
+		{name: "region A again", bbox: regionA, wantCalls: 1, wantFiles: "a.gml"},
+		{name: "region A after a tile is added in region B", bbox: regionA, add: "c.gml", wantCalls: 1, wantFiles: "a.gml"},
+		{name: "region B", bbox: regionB, wantCalls: 2, wantFiles: "b.gml c.gml"},
+		{name: "a bbox no file reaches", bbox: elsewhere, wantCalls: 2, wantFiles: "b.gml c.gml"},
+	}
+	for _, s := range steps {
+		if s.add != "" {
+			write(s.add, 792000, 5416000)
+		}
+		if err := importer.ImportCityDBData(testPool, cfg, s.bbox, "intersects"); err != nil {
+			t.Fatalf("%s: ImportCityDBData: %v", s.name, err)
+		}
+		if got := importCalls(t, logPath); got != s.wantCalls {
+			t.Fatalf("%s: citydb import called %d times in total, want %d", s.name, got, s.wantCalls)
+		}
+		if got := lastFiles(t, logPath); got != s.wantFiles {
+			t.Fatalf("%s: last import got files %q, want %q", s.name, got, s.wantFiles)
+		}
+	}
+}