duckdb catalog: auto-detect rel_* tables as relationship tables

duckdb/src/catalog/duckdb_catalog.cpp

@@ -1,10 +1,14 @@
 #include "catalog/duckdb_catalog.h"
 
+#include <regex>
+
 #include "binder/bound_attach_info.h"
 #include "catalog/catalog_entry/node_table_catalog_entry.h"
+#include "catalog/catalog_entry/rel_group_catalog_entry.h"
 #include "catalog/duckdb_table_catalog_entry.h"
 #include "common/exception/binder.h"
 #include "common/exception/runtime.h"
+#include "common/string_utils.h"
 #include "connector/duckdb_type_converter.h"
 #include "function/duckdb_scan.h"
 #include "storage/buffer_manager/memory_manager.h"
@@ -53,10 +57,35 @@ void DuckDBCatalog::init() {
     DuckDBResultConverter::getDuckDBVectorConversionFunc(common::PhysicalTypeID::STRING,
         conversionFunc);
     conversionFunc(resultChunk->data[0], tableNamesVector, resultChunk->size());
+    // Two-pass initialization: node tables must be registered before rel tables
+    // so that rel tables can resolve their src/dst node table IDs. The table
+    // enumeration order is alphabetical, which can put rel_* tables before the
+    // node tables they reference.
+    // First pass: register node tables (everything that is not a rel table).
     for (auto i = 0u; i < resultChunk->size(); i++) {
         auto tableName = tableNamesVector.getValue<common::string_t>(i).getAsString();
+        auto lowerName = tableName;
+        common::StringUtils::toLower(lowerName);
+        if (lowerName.rfind("rel_", 0) == 0 || lowerName.rfind("csr_rel_", 0) == 0) {
+            continue;
+        }
         createForeignTable(tableName);
     }
+    // Second pass: register rel tables.
+    for (auto i = 0u; i < resultChunk->size(); i++) {
+        auto tableName = tableNamesVector.getValue<common::string_t>(i).getAsString();
+        auto lowerName = tableName;
+        common::StringUtils::toLower(lowerName);
+        if (lowerName.rfind("rel_", 0) == 0) {
+            // Foreign-key-based rel table: scan-driven, optimizer generates a join.
+            // No CSR columns; backed by a ForeignRelTable.
+            createForeignRelTable(tableName);
+        } else if (lowerName.rfind("csr_rel_", 0) == 0) {
+            // CSR-based rel table: materialized into a local on-disk CSR rel table.
+            // TODO: COPY data from DuckDB into a local RelTable.
+            createForeignRelTable(tableName);
+        }
+    }
 }
 
 std::string DuckDBCatalog::bindSchemaName(const binder::AttachOption& options,
@@ -118,6 +147,118 @@ void DuckDBCatalog::createForeignTable(const std::string& tableName) {
     lbug::storage::StorageManager::Get(*context_)->createTable(mainEntry);
 }
 
+void DuckDBCatalog::createForeignRelTable(const std::string& tableName) {
+    // Query foreign key info to find src/dst node tables.
+    //
+    // information_schema.constraint_column_usage is unusable for this in
+    // DuckDB: for FK constraints it reports the constraint's own table (the
+    // referencing side), not the referenced one. duckdb_constraints() exposes
+    // the referenced table directly; unnest() flattens the column list so each
+    // row yields (fk_column, referenced_table).
+    auto fkQuery = std::format("SELECT unnest(constraint_column_names) AS column_name, "
+                               "referenced_table FROM duckdb_constraints() "
+                               "WHERE constraint_type = 'FOREIGN KEY' "
+                               "AND referenced_table IS NOT NULL AND table_name = '{}'",
+        tableName);
+    auto fkResult = connector.executeQuery(fkQuery);
+
+    std::string srcTableName, dstTableName;
+    for (auto i = 0u; i < fkResult->RowCount(); i++) {
+        auto colName = fkResult->GetValue(0, i).GetValue<std::string>();
+        auto refTable = fkResult->GetValue(1, i).GetValue<std::string>();
+        auto lowerCol = colName;
+        common::StringUtils::toLower(lowerCol);
+        if (lowerCol == "src_id" || lowerCol.find("src") == 0) {
+            srcTableName = refTable;
+        } else if (lowerCol == "dst_id" || lowerCol.find("dst") == 0 ||
+                   lowerCol.find("dest") == 0) {
+            dstTableName = refTable;
+        }
+    }
+
+    if (srcTableName.empty() || dstTableName.empty()) {
+        createForeignTable(tableName);
+        return;
+    }
+
+    // Build property definitions
+    std::vector<binder::PropertyDefinition> propertyDefinitions;
+    bindPropertyDefinitions(tableName, propertyDefinitions);
+
+    // Determine the node table IDs from the main catalog. containsTable() must
+    // be checked first: getTableCatalogEntry() throws when the table is
+    // missing, and a rel table may reference tables that were not registered.
+    auto* catalog = context_->getDatabase()->getCatalog();
+    if (!catalog->containsTable(&transaction::DUMMY_TRANSACTION, srcTableName) ||
+        !catalog->containsTable(&transaction::DUMMY_TRANSACTION, dstTableName)) {
+        createForeignTable(tableName);
+        return;
+    }
+    auto* srcEntry = catalog->getTableCatalogEntry(&transaction::DUMMY_TRANSACTION, srcTableName);
+    auto* dstEntry = catalog->getTableCatalogEntry(&transaction::DUMMY_TRANSACTION, dstTableName);
+    if (srcEntry == nullptr || dstEntry == nullptr) {
+        createForeignTable(tableName);
+        return;
+    }
+
+    common::table_id_t srcTableID = srcEntry->getTableID();
+    common::table_id_t dstTableID = dstEntry->getTableID();
+
+    // Build query and scan info
+    std::vector<common::LogicalType> columnTypes;
+    std::vector<std::string> columnNames;
+    for (auto& def : propertyDefinitions) {
+        columnNames.push_back(def.getName());
+        columnTypes.push_back(def.getType().copy());
+    }
+
+    auto queryStr =
+        std::format("SELECT * FROM \"{}\".{}.{}", catalogName, defaultSchemaName, tableName);
+    auto duckdbTableInfo = std::make_shared<DuckDBTableScanInfo>(queryStr, std::move(columnTypes),
+        columnNames, connector);
+    auto scanFunc = getScanFunction(duckdbTableInfo);
+
+    // Create DuckDB table catalog entry
+    auto tableEntry =
+        std::make_unique<catalog::DuckDBTableCatalogEntry>(tableName, scanFunc, duckdbTableInfo);
+    for (auto& def : propertyDefinitions) {
+        tableEntry->addProperty(def);
+    }
+    tables->createEntry(&transaction::DUMMY_TRANSACTION, std::move(tableEntry));
+
+    // Create bind data for the scan function
+    binder::expression_vector emptyColumns;
+    auto bindData =
+        std::make_shared<DuckDBScanBindData>(queryStr, columnNames, connector, emptyColumns);
+
+    // Create RelGroupCatalogEntry
+    auto foreignDatabaseName = std::format("{}.{}", catalogName, tableName);
+
+    std::vector<catalog::RelTableCatalogInfo> relTableInfos;
+    auto info = bindCreateTableInfo(tableName);
+    common::oid_t relOID = tables->getNextOID();
+    relTableInfos.emplace_back(catalog::NodeTableIDPair{srcTableID, dstTableID}, relOID,
+        common::RelMultiplicity::MANY, common::RelMultiplicity::MANY);
+
+    auto relGroupEntry =
+        std::make_unique<catalog::RelGroupCatalogEntry>(tableName, common::RelMultiplicity::MANY,
+            common::RelMultiplicity::MANY, common::ExtendDirection::BOTH, std::move(relTableInfos),
+            "", // storage
+            common::StorageFormat::NONE, scanFunc, bindData, std::move(foreignDatabaseName));
+
+    for (auto& def : propertyDefinitions) {
+        relGroupEntry->addProperty(def);
+    }
+
+    context_->getDatabase()->getCatalog()->addTableEntry(std::move(relGroupEntry));
+
+    auto mainEntry = context_->getDatabase()->getCatalog()->getTableCatalogEntry(
+        &transaction::DUMMY_TRANSACTION, tableName);
+    if (mainEntry) {
+        storage::StorageManager::Get(*context_)->createTable(mainEntry);
+    }
+}
+
 static bool getTableInfo(const DuckDBConnector& connector, const std::string& tableName,
     const std::string& schemaName, const std::string& catalogName,
     std::vector<common::LogicalType>& columnTypes, std::vector<std::string>& columnNames,

duckdb/src/include/catalog/duckdb_catalog.h

@@ -50,6 +50,7 @@ class DuckDBCatalog : public extension::CatalogExtension {
 
 private:
     void createForeignTable(const std::string& tableName);
+    void createForeignRelTable(const std::string& tableName);
 
 protected:
     std::string dbPath;

duckdb/test/test_files/duckdb_rel.test

@@ -0,0 +1,64 @@
+-DATASET CSV empty
+
+--
+
+-CASE DuckdbAutoDetectRelTables
+-SKIP_FSM_LEAK_CHECK
+-LOAD_DYNAMIC_EXTENSION duckdb
+-STATEMENT ATTACH '${LBUG_ROOT_DIRECTORY}/extension/duckdb/test/test_files/users_sessions.db' as g (dbtype duckdb);
+---- 1
+Attached database successfully.
+-STATEMENT LOAD FROM g.users RETURN name ORDER BY name;
+---- 3
+Alice
+Bob
+Carol
+-STATEMENT LOAD FROM g.sessions RETURN device ORDER BY device;
+---- 4
+desktop
+laptop
+mobile
+tablet
+-STATEMENT LOAD FROM g.rel_user_owns_session RETURN src_user, dst_session, since ORDER BY src_user, dst_session;
+---- 5
+0|1|2024-01-15
+0|2|2024-02-20
+1|0|2024-03-05
+2|1|2024-05-25
+2|3|2024-04-10
+-STATEMENT MATCH (u:g.users) RETURN count(*);
+---- 1
+3
+-STATEMENT MATCH (u:g.users) WHERE u.name = 'Alice' RETURN count(*);
+---- 1
+1
+-STATEMENT DETACH g;
+---- ok
+
+-CASE DuckdbCsrRelPrefixRegistersAsRelTable
+-SKIP_FSM_LEAK_CHECK
+-LOAD_DYNAMIC_EXTENSION duckdb
+-STATEMENT ATTACH '${LBUG_ROOT_DIRECTORY}/extension/duckdb/test/test_files/users_sessions.db' as g (dbtype duckdb);
+---- 1
+Attached database successfully.
+-STATEMENT LOAD FROM g.csr_rel_user_blocks_user RETURN src_user, dst_blocked ORDER BY src_user, dst_blocked;
+---- 2
+0|1
+2|1
+-STATEMENT MATCH (u:g.users) RETURN count(*);
+---- 1
+3
+-STATEMENT DETACH g;
+---- ok
+
+-CASE DuckdbCsrRelTraversalFailsFast
+-SKIP_FSM_LEAK_CHECK
+-LOAD_DYNAMIC_EXTENSION duckdb
+-STATEMENT ATTACH '${LBUG_ROOT_DIRECTORY}/extension/duckdb/test/test_files/users_sessions.db' as g (dbtype duckdb);
+---- 1
+Attached database successfully.
+-STATEMENT MATCH (u:users)-[b:csr_rel_user_blocks_user]->(v:users) RETURN count(*);
+---- error
+Runtime exception: MATCH traversal over foreign-backed rel table "csr_rel_user_blocks_user" is not supported yet: the foreign key to internal ID mapping layer is not implemented
+-STATEMENT DETACH g;
+---- ok