diff --git a/CHANGELOG.md b/CHANGELOG.md index 98daeccf..530f08a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] +- +- Add DuckDB BaseConverter for efficiently transforming large datasets ## [v0.21.0] - 2026-02-16 diff --git a/fiboa_cli/conversion/duckdb.py b/fiboa_cli/conversion/duckdb.py new file mode 100644 index 00000000..9b4f9fc1 --- /dev/null +++ b/fiboa_cli/conversion/duckdb.py @@ -0,0 +1,221 @@ +import json +import os +from pathlib import Path +from tempfile import NamedTemporaryFile + +import duckdb +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +from geopandas.array import from_wkb +from pyarrow.lib import StructArray +from vecorel_cli.encoding.geojson import VecorelJSONEncoder + +from .fiboa_converter import FiboaBaseConverter + + +# This converter is experimental, use with caution. +# Results may not be fully fiboa compliant yet. +# Use this primarily for datasets that are too large to be processed by the default converter +class FiboaDuckDBBaseConverter(FiboaBaseConverter): + def convert( + self, + output_file, + cache=None, + input_files=None, + variant=None, + compression=None, + geoparquet_version=None, + original_geometries=False, + **kwargs, + ) -> str: + if not original_geometries: + self.warning( + "original_geometries is not supported for DuckDB-based converters and will always write original geometries" + ) + + geoparquet_version = geoparquet_version or "1.1.0" + compression = compression or "brotli" + + self.variant = variant + cid = self.id.strip() + if self.bbox is not None and len(self.bbox) != 4: + raise ValueError("If provided, the bounding box must consist of 4 numbers") + + # Create output folder if it doesn't exist + directory = os.path.dirname(output_file) + if directory: + os.makedirs(directory, exist_ok=True) + + if input_files is not None and isinstance(input_files, dict) and len(input_files) > 0: + self.warning("Using user provided input file(s) instead of the pre-defined file(s)") + urls = input_files + else: + urls = self.get_urls() + if urls is None: + raise ValueError("No input files provided") + + self.info("Getting file(s) if not cached yet") + if cache: + request_args = {} + if self.avoid_range_request: + request_args["block_size"] = 0 + urls = self.download_files(urls, cache, **request_args) + elif self.avoid_range_request: + self.warning( + "avoid_range_request is set, but cache is not used, so this setting has no effect" + ) + + selections = [] + geom_column = None + for k, v in self.columns.items(): + if k in self.column_migrations: + selections.append(f'{self.column_migrations.get(k)} as "{v}"') + else: + selections.append(f'"{k}" as "{v}"') + if v == "geometry": + geom_column = k + selection = ", ".join(selections) + + filters = [] + where = "" + if self.bbox is not None: + filters.append( + f"ST_Intersects(geometry, ST_MakeEnvelope({self.bbox[0]}, {self.bbox[1]}, {self.bbox[2]}, {self.bbox[3]}))" + ) + for k, v in self.column_filters.items(): + filters.append(v) + if len(filters) > 0: + where = f"WHERE {' AND '.join(filters)}" + + if isinstance(urls, str): + sources = f'"{urls}"' + else: + paths = [] + for url in urls: + if isinstance(url, tuple): + paths.append(f'"{url[0]}"') + else: + paths.append(f'"{url}"') + sources = "[" + ",".join(paths) + "]" + + collection = self.create_collection(cid) + collection.update(self.column_additions) + collection["collection"] = self.id + + if isinstance(output_file, Path): + output_file = str(output_file) + + collection_json = json.dumps(collection, cls=VecorelJSONEncoder).encode("utf-8") + + con = duckdb.connect() + con.install_extension("spatial") + con.load_extension("spatial") + con.execute( + f""" + COPY ( + SELECT {selection} + FROM read_parquet({sources}, union_by_name=true) + {where} + ORDER BY ST_Hilbert({geom_column}) + ) TO ? ( + FORMAT parquet, + compression ?, + KV_METADATA {{ + collection: ?, + }} + ) + """, + [output_file, compression, collection_json], + ) + + # Post-process the written Parquet to proper GeoParquet v1.1 with bbox and nullability + try: + pq_file = pq.ParquetFile(output_file) + + existing_schema = pq_file.schema_arrow + col_names = existing_schema.names + assert "geometry" in col_names, "Missing geometry column in output parquet file" + + schemas = collection.merge_schemas({}) + collection_only = {k for k, v in schemas.get("collection", {}).items() if v} + required_columns = {"geometry"} | { + r + for r in schemas.get("required", []) + if r in col_names and r not in collection_only + } + if "id" in col_names: + required_columns.add("id") + + # Update for version 1.1.0 + metadata = existing_schema.metadata + if geoparquet_version > "1.0.0": + geo_meta = json.loads(existing_schema.metadata[b"geo"]) + geo_meta["version"] = geoparquet_version + metadata[b"geo"] = json.dumps(geo_meta).encode("utf-8") + + # Build a new Arrow schema with adjusted nullability + new_fields = [] + for field in existing_schema: + if field.name in required_columns and field.nullable: + new_fields.append( + pa.field(field.name, field.type, nullable=False, metadata=field.metadata) + ) + else: + new_fields.append(field) + + add_bbox = geoparquet_version > "1.0.0" and "bbox" not in col_names + if add_bbox: + new_fields.append( + pa.field( + "bbox", + pa.struct( + [ + ("xmin", pa.float64()), + ("ymin", pa.float64()), + ("xmax", pa.float64()), + ("ymax", pa.float64()), + ] + ), + ) + ) + new_schema = pa.schema(new_fields, metadata=metadata) + + # 7) Streamingly rewrite the file to a temp file and replace atomically + with NamedTemporaryFile( + "wb", delete=False, dir=os.path.dirname(output_file), suffix=".parquet" + ) as tmp: + tmp_path = tmp.name + + writer = pq.ParquetWriter( + tmp_path, + new_schema, + compression=compression, + use_dictionary=True, + write_statistics=True, + ) + try: + bbox_names = ["ymax", "xmax", "ymin", "xmin"] + for rg in range(pq_file.num_row_groups): + tbl = pq_file.read_row_group(rg) + if add_bbox: + # determine bounds, change to StructArray type + bounds = from_wkb(tbl["geometry"]).bounds + bbox_array = StructArray.from_arrays( + np.rot90(bounds), + names=bbox_names, + ) + tbl = tbl.append_column("bbox", bbox_array) + # Ensure table adheres to the new schema (mainly nullability); cast if needed + if tbl.schema != new_schema: + # Align field order/types; this does not materialize data beyond the batch + tbl = tbl.cast(new_schema, safe=False) + writer.write_table(tbl) + finally: + writer.close() + + os.replace(tmp_path, output_file) + except Exception as e: + self.warning(f"GeoParquet 1.1 post-processing failed: {e}") + + return output_file diff --git a/fiboa_cli/datasets/jp.py b/fiboa_cli/datasets/jp.py index 57c31a24..1799c7f4 100644 --- a/fiboa_cli/datasets/jp.py +++ b/fiboa_cli/datasets/jp.py @@ -1,10 +1,9 @@ -import pandas as pd +from fiboa_cli.conversion.duckdb import FiboaDuckDBBaseConverter -from ..conversion.fiboa_converter import FiboaBaseConverter - -class JPConverter(FiboaBaseConverter): +class JPConverter(FiboaDuckDBBaseConverter): variants = { + "test": "./tests/data-files/convert/jp/jp_field_polygons_2024.parquet", "2024": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2024.parquet", "2023": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2023.parquet", "2022": "https://data.source.coop/pacificspatial/field-polygon-jp/parquet/jp_field_polygons_2022.parquet", @@ -30,23 +29,11 @@ class JPConverter(FiboaBaseConverter): "polygon_uuid": "id", "land_type_en": "land_type_en", "local_government_cd": "admin_local_code", - "issue_year": "determination:datetime", - } - column_migrations = { - "issue_year": lambda col: pd.to_datetime(col, format="%Y"), } - + column_additions = {"determination:datetime": "2024-01-01T00:00:00Z"} missing_schemas = { "properties": { "land_type_en": {"type": "string"}, "admin_local_code": {"type": "string"}, } } - - def convert(self, *args, **kwargs): - # Open only these columns to limit memory usage - super().convert( - *args, - columns=["GEOM", "polygon_uuid", "land_type_en", "local_government_cd", "issue_year"], - **kwargs, - ) diff --git a/pixi.lock b/pixi.lock index 1f6b2a69..fee018c9 100644 --- a/pixi.lock +++ b/pixi.lock @@ -141,6 +141,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/35/b1fae4c5245697837f6f63e407fa81e7ccc7948f6ef2b124cd38736f4d1d/duckdb-1.4.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/90/72/0b6035302e9c33f004240a50cb6e2e1fc7bb1f2b415b02d939c551bdd06b/inflate64-1.0.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl @@ -302,6 +303,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h3eecb57_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/ab/e04a8f97865251b544aee9501088d4f0cb8e8b37339bd465c0d33857d411/duckdb-1.4.2-cp314-cp314-macosx_10_15_x86_64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/81/80/24ba0d2ee14e07e275e9c5b058e59a8a58f8ef42dd51a78ebbfd7c857ac4/inflate64-1.0.4-cp314-cp314-macosx_10_15_x86_64.whl @@ -464,6 +466,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/47/ec/b8229517c2f9fe88a38bb1a172a2da4d0ff34996d319d74554fda80b6358/duckdb-1.4.2-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/b8/073a79716e093db973b8823bdfb02e10fbdf65642dbe1fa3cda24832aeb2/inflate64-1.0.4-cp314-cp314-macosx_11_0_arm64.whl @@ -624,6 +627,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-h534d264_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/25/5e/6f5ebaabc12c6db62f471f86b5c9c8debd57f11aa1b2acbbcc4c68683238/duckdb-1.4.2-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dd/51/f2972df8cceecc9bf3afa3353d517ffc7125285198c844588e9aaf98f5d0/inflate64-1.0.4-cp314-cp314-win_amd64.whl @@ -777,6 +781,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/35/b1fae4c5245697837f6f63e407fa81e7ccc7948f6ef2b124cd38736f4d1d/duckdb-1.4.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/90/72/0b6035302e9c33f004240a50cb6e2e1fc7bb1f2b415b02d939c551bdd06b/inflate64-1.0.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl @@ -913,6 +918,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h3eecb57_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/ab/e04a8f97865251b544aee9501088d4f0cb8e8b37339bd465c0d33857d411/duckdb-1.4.2-cp314-cp314-macosx_10_15_x86_64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/81/80/24ba0d2ee14e07e275e9c5b058e59a8a58f8ef42dd51a78ebbfd7c857ac4/inflate64-1.0.4-cp314-cp314-macosx_10_15_x86_64.whl @@ -1050,6 +1056,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/47/ec/b8229517c2f9fe88a38bb1a172a2da4d0ff34996d319d74554fda80b6358/duckdb-1.4.2-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/b8/073a79716e093db973b8823bdfb02e10fbdf65642dbe1fa3cda24832aeb2/inflate64-1.0.4-cp314-cp314-macosx_11_0_arm64.whl @@ -1186,6 +1193,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-h534d264_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/25/5e/6f5ebaabc12c6db62f471f86b5c9c8debd57f11aa1b2acbbcc4c68683238/duckdb-1.4.2-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dd/51/f2972df8cceecc9bf3afa3353d517ffc7125285198c844588e9aaf98f5d0/inflate64-1.0.4-cp314-cp314-win_amd64.whl @@ -1300,6 +1308,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/35/b1fae4c5245697837f6f63e407fa81e7ccc7948f6ef2b124cd38736f4d1d/duckdb-1.4.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/90/72/0b6035302e9c33f004240a50cb6e2e1fc7bb1f2b415b02d939c551bdd06b/inflate64-1.0.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl @@ -1399,6 +1408,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h3eecb57_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/ab/e04a8f97865251b544aee9501088d4f0cb8e8b37339bd465c0d33857d411/duckdb-1.4.2-cp314-cp314-macosx_10_15_x86_64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/81/80/24ba0d2ee14e07e275e9c5b058e59a8a58f8ef42dd51a78ebbfd7c857ac4/inflate64-1.0.4-cp314-cp314-macosx_10_15_x86_64.whl @@ -1499,6 +1509,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/47/ec/b8229517c2f9fe88a38bb1a172a2da4d0ff34996d319d74554fda80b6358/duckdb-1.4.2-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/b8/073a79716e093db973b8823bdfb02e10fbdf65642dbe1fa3cda24832aeb2/inflate64-1.0.4-cp314-cp314-macosx_11_0_arm64.whl @@ -1596,6 +1607,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/yarl-1.22.0-pyh7db6752_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-h534d264_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/25/5e/6f5ebaabc12c6db62f471f86b5c9c8debd57f11aa1b2acbbcc4c68683238/duckdb-1.4.2-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dd/51/f2972df8cceecc9bf3afa3353d517ffc7125285198c844588e9aaf98f5d0/inflate64-1.0.4-cp314-cp314-win_amd64.whl @@ -1736,6 +1748,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/23/35/b1fae4c5245697837f6f63e407fa81e7ccc7948f6ef2b124cd38736f4d1d/duckdb-1.4.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/90/72/0b6035302e9c33f004240a50cb6e2e1fc7bb1f2b415b02d939c551bdd06b/inflate64-1.0.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl @@ -1862,6 +1875,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/zstd-1.5.7-h3eecb57_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/ab/e04a8f97865251b544aee9501088d4f0cb8e8b37339bd465c0d33857d411/duckdb-1.4.2-cp314-cp314-macosx_10_15_x86_64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/81/80/24ba0d2ee14e07e275e9c5b058e59a8a58f8ef42dd51a78ebbfd7c857ac4/inflate64-1.0.4-cp314-cp314-macosx_10_15_x86_64.whl @@ -1989,6 +2003,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/47/ec/b8229517c2f9fe88a38bb1a172a2da4d0ff34996d319d74554fda80b6358/duckdb-1.4.2-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/b8/073a79716e093db973b8823bdfb02e10fbdf65642dbe1fa3cda24832aeb2/inflate64-1.0.4-cp314-cp314-macosx_11_0_arm64.whl @@ -2112,6 +2127,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.7-h534d264_6.conda - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c5/0d/84a4380f930db0010168e0aa7b7a8fed9ba1835a8fbb1472bc6d0201d529/build-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/25/5e/6f5ebaabc12c6db62f471f86b5c9c8debd57f11aa1b2acbbcc4c68683238/duckdb-1.4.2-cp314-cp314-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/54/e4/fac19dc34cb686c96011388b813ff7b858a70681e5ce6ce7698e5021b0f4/geopandas-1.1.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dd/51/f2972df8cceecc9bf3afa3353d517ffc7125285198c844588e9aaf98f5d0/inflate64-1.0.4-cp314-cp314-win_amd64.whl @@ -2789,6 +2805,54 @@ packages: - pkg:pypi/distlib?source=hash-mapping size: 275642 timestamp: 1752823081585 +- pypi: https://files.pythonhosted.org/packages/23/35/b1fae4c5245697837f6f63e407fa81e7ccc7948f6ef2b124cd38736f4d1d/duckdb-1.4.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl + name: duckdb + version: 1.4.2 + sha256: 128c97dab574a438d7c8d020670b21c68792267d88e65a7773667b556541fa9b + requires_dist: + - ipython ; extra == 'all' + - fsspec ; extra == 'all' + - numpy ; extra == 'all' + - pandas ; extra == 'all' + - pyarrow ; extra == 'all' + - adbc-driver-manager ; extra == 'all' + requires_python: '>=3.9.0' +- pypi: https://files.pythonhosted.org/packages/25/5e/6f5ebaabc12c6db62f471f86b5c9c8debd57f11aa1b2acbbcc4c68683238/duckdb-1.4.2-cp314-cp314-win_amd64.whl + name: duckdb + version: 1.4.2 + sha256: dfcc56a83420c0dec0b83e97a6b33addac1b7554b8828894f9d203955591218c + requires_dist: + - ipython ; extra == 'all' + - fsspec ; extra == 'all' + - numpy ; extra == 'all' + - pandas ; extra == 'all' + - pyarrow ; extra == 'all' + - adbc-driver-manager ; extra == 'all' + requires_python: '>=3.9.0' +- pypi: https://files.pythonhosted.org/packages/47/ec/b8229517c2f9fe88a38bb1a172a2da4d0ff34996d319d74554fda80b6358/duckdb-1.4.2-cp314-cp314-macosx_11_0_arm64.whl + name: duckdb + version: 1.4.2 + sha256: 20c45b4ead1ea4d23a1be1cd4f1dfc635e58b55f0dd11e38781369be6c549903 + requires_dist: + - ipython ; extra == 'all' + - fsspec ; extra == 'all' + - numpy ; extra == 'all' + - pandas ; extra == 'all' + - pyarrow ; extra == 'all' + - adbc-driver-manager ; extra == 'all' + requires_python: '>=3.9.0' +- pypi: https://files.pythonhosted.org/packages/82/ab/e04a8f97865251b544aee9501088d4f0cb8e8b37339bd465c0d33857d411/duckdb-1.4.2-cp314-cp314-macosx_10_15_x86_64.whl + name: duckdb + version: 1.4.2 + sha256: 459b1855bd06a226a2838da4f14c8863fd87a62e63d414a7f7f682a7c616511a + requires_dist: + - ipython ; extra == 'all' + - fsspec ; extra == 'all' + - numpy ; extra == 'all' + - pandas ; extra == 'all' + - pyarrow ; extra == 'all' + - adbc-driver-manager ; extra == 'all' + requires_python: '>=3.9.0' - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.3.1-pyhd8ed1ab_0.conda sha256: ee6cf346d017d954255bbcbdb424cddea4d14e4ed7e9813e429db1d795d01144 md5: 8e662bd460bda79b1ea39194e3c4c9ab @@ -2803,10 +2867,11 @@ packages: - pypi: ./ name: fiboa-cli version: 0.21.0 - sha256: 5a8a4c3d9234870ac9a54c0cc925c41f59ca033ee111dd241c1af9633591f47b + sha256: 544a959421501ea1dcf28cede8c6b5c41b70ce5ddeb55f0c4ed1a54fc32c2d64 requires_dist: - vecorel-cli==0.2.15 - spdx-license-list==3.27.0 + - duckdb==1.4.2 requires_python: '>=3.11' editable: true - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.21.2-pyhd8ed1ab_0.conda diff --git a/pyproject.toml b/pyproject.toml index 735559ce..7e56f4f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ requires-python = ">=3.11" dependencies = [ "vecorel-cli==0.2.15", "spdx-license-list==3.27.0", + "duckdb==1.4.2", ] [project.scripts]