fix: escape relation string literals in metadata queries (#1747)

CHANGELOG.md

@@ -8,6 +8,8 @@
 
 ### Fixes
 
+- Escape single quotes and backslashes in relation names inside metadata queries to prevent syntax errors on Unity Catalog relations containing special characters (resolves [#1747](https://github.com/databricks/dbt-databricks/issues/1747))
+
 - Stop `view_update_via_alter` failing with `UNRESOLVED_COLUMN` when a query change drops a column whose description also changed, and keep column comments after query changes on views inheriting a non-Delta `file_format` ([#1736](https://github.com/databricks/dbt-databricks/pull/1736))
 - Enforce model contracts and reject column masks and row filters when `view_update_via_alter` alters a view in place or finds no changes, matching `CREATE` ([#1736](https://github.com/databricks/dbt-databricks/pull/1736))
 - Replace views with `CREATE OR REPLACE VIEW` when `use_safer_relation_operations` is on, including views inheriting `file_format: parquet`, so upstream column drops or renames no longer fail with `UNRESOLVED_COLUMN` (thanks @shantanutheone!) ([#1733](https://github.com/databricks/dbt-databricks/pull/1733) resolves [#1714](https://github.com/databricks/dbt-databricks/issues/1714))

dbt/include/databricks/macros/adapters/catalog.sql

@@ -65,24 +65,24 @@ ORDER BY column_index
 {%- endmacro %}
 
 {% macro databricks__get_catalog_schemas_where_clause_sql(catalog, schemas) -%}
-WHERE table_catalog = '{{ catalog|lower }}' AND (
+WHERE table_catalog = '{{ escape_relation_literal(catalog|lower) }}' AND (
   {%- for relation in schemas -%}
-  table_schema = '{{ relation[1]|lower }}'{%- if not loop.last %} OR {% endif -%}
+  table_schema = '{{ escape_relation_literal(relation[1]|lower) }}'{%- if not loop.last %} OR {% endif -%}
   {%- endfor -%})
 {%- endmacro %}
 
 
 {% macro databricks__get_catalog_relations_where_clause_sql(catalog, relations) -%}
-WHERE table_catalog = '{{ catalog|lower }}' AND (
+WHERE table_catalog = '{{ escape_relation_literal(catalog|lower) }}' AND (
   {%- for relation in relations -%}
     {%- if relation.schema and relation.identifier %}
   (
-    table_schema = '{{ relation.schema|lower }}'
-    AND table_name = '{{ relation.identifier|lower }}'
+    table_schema = '{{ escape_relation_literal(relation.schema|lower) }}'
+    AND table_name = '{{ escape_relation_literal(relation.identifier|lower) }}'
   )
     {%- elif relation.schema %}
   (
-    table_schema = '{{ relation.schema|lower }}'
+    table_schema = '{{ escape_relation_literal(relation.schema|lower) }}'
   )
     {% else %}
       {% do exceptions.raise_compiler_error(

dbt/include/databricks/macros/adapters/metadata.sql

@@ -70,8 +70,8 @@ SHOW VIEWS IN {{ relation.render() }}
   {% if information_schema.is_hive_metastore() %}
     {%- for relation in relations -%}
 SELECT
-  '{{ relation.schema }}' AS schema,
-  '{{ relation.identifier }}' AS identifier,
+  '{{ escape_relation_literal(relation.schema) }}' AS schema,
+  '{{ escape_relation_literal(relation.identifier) }}' AS identifier,
   max(timestamp) AS last_modified,
   {{ current_timestamp() }} AS snapshotted_at
   FROM (DESCRIBE HISTORY {{ relation.render() }})
@@ -86,11 +86,11 @@ SELECT
   last_altered AS last_modified,
   {{ current_timestamp() }} AS snapshotted_at
 FROM `system`.`information_schema`.`tables`
-WHERE table_catalog = '{{ information_schema.database|lower }}'
+WHERE table_catalog = '{{ escape_relation_literal(information_schema.database|lower) }}'
   AND (
     {%- for relation in relations -%}
-    (table_schema = '{{ relation.schema|lower }}' AND
-    table_name = '{{ relation.identifier|lower }}'){%- if not loop.last %} OR {% endif -%}
+    (table_schema = '{{ escape_relation_literal(relation.schema|lower) }}' AND
+    table_name = '{{ escape_relation_literal(relation.identifier|lower) }}'){%- if not loop.last %} OR {% endif -%}
     {%- endfor -%}
   )
   {% endif %}
@@ -103,9 +103,9 @@ WHERE table_catalog = '{{ information_schema.database|lower }}'
 {% macro get_view_description_sql(relation) %}
 SELECT *
 FROM `system`.`information_schema`.`views`
-WHERE table_catalog = '{{ relation.database|lower }}'
-  AND table_schema = '{{ relation.schema|lower }}'
-  AND table_name = '{{ relation.identifier|lower }}'
+WHERE table_catalog = '{{ escape_relation_literal(relation.database|lower) }}'
+  AND table_schema = '{{ escape_relation_literal(relation.schema|lower) }}'
+  AND table_name = '{{ escape_relation_literal(relation.identifier|lower) }}'
 {% endmacro %}
 
 {% macro get_uc_tables(relation) %}
@@ -129,10 +129,10 @@ SELECT
     NULL
   ) AS databricks_table_type
 FROM `system`.`information_schema`.`tables`
-WHERE table_catalog = '{{ relation.database|lower }}'
-  AND table_schema = '{{ relation.schema|lower }}'
+WHERE table_catalog = '{{ escape_relation_literal(relation.database|lower) }}'
+  AND table_schema = '{{ escape_relation_literal(relation.schema|lower) }}'
   {%- if relation.identifier %}
-  AND table_name = '{{ relation.identifier|lower }}'
+  AND table_name = '{{ escape_relation_literal(relation.identifier|lower) }}'
   {% endif %}
 {% endmacro %}
 

dbt/include/databricks/macros/relations/components/column_mask.sql

@@ -14,9 +14,9 @@
     mask_name,
     using_columns
   FROM `system`.`information_schema`.`column_masks`
-  WHERE table_catalog = '{{ relation.database|lower }}'
-    AND table_schema = '{{ relation.schema|lower }}'
-    AND table_name = '{{ relation.identifier|lower }}';
+  WHERE table_catalog = '{{ escape_relation_literal(relation.database|lower) }}'
+    AND table_schema = '{{ escape_relation_literal(relation.schema|lower) }}'
+    AND table_name = '{{ escape_relation_literal(relation.identifier|lower) }}';
 {%- endmacro -%}
 
 {% macro apply_column_masks(relation, column_masks) -%}

dbt/include/databricks/macros/relations/components/column_tags.sql

@@ -14,9 +14,9 @@
     tag_name,
     tag_value
   FROM `system`.`information_schema`.`column_tags`
-  WHERE catalog_name = '{{ relation.database|lower }}'
-    AND schema_name = '{{ relation.schema|lower }}'
-    AND table_name = '{{ relation.identifier|lower }}';
+  WHERE catalog_name = '{{ escape_relation_literal(relation.database|lower) }}'
+    AND schema_name = '{{ escape_relation_literal(relation.schema|lower) }}'
+    AND table_name = '{{ escape_relation_literal(relation.identifier|lower) }}';
 {%- endmacro -%}
 
 {% macro apply_column_tags(relation, column_tags) -%}

dbt/include/databricks/macros/relations/components/constraints.sql

@@ -11,9 +11,9 @@
 {% macro fetch_non_null_constraint_columns_sql(relation) -%}
   SELECT column_name
   FROM `{{ relation.database|lower }}`.`information_schema`.`columns`
-  WHERE table_catalog = '{{ relation.database|lower }}' 
-    AND table_schema = '{{ relation.schema|lower }}'
-    AND table_name = '{{ relation.identifier|lower }}'
+  WHERE table_catalog = '{{ escape_relation_literal(relation.database|lower) }}' 
+    AND table_schema = '{{ escape_relation_literal(relation.schema|lower) }}'
+    AND table_name = '{{ escape_relation_literal(relation.identifier|lower) }}'
     AND is_nullable = 'NO';
 {%- endmacro -%}
 
@@ -30,15 +30,15 @@
 {% macro fetch_primary_key_constraints_sql(relation) -%}
   SELECT kcu.constraint_name, kcu.column_name
   FROM `{{ relation.database|lower }}`.information_schema.key_column_usage kcu
-  WHERE kcu.table_catalog = '{{ relation.database|lower }}' 
-    AND kcu.table_schema = '{{ relation.schema|lower }}'
-    AND kcu.table_name = '{{ relation.identifier|lower }}' 
+  WHERE kcu.table_catalog = '{{ escape_relation_literal(relation.database|lower) }}' 
+    AND kcu.table_schema = '{{ escape_relation_literal(relation.schema|lower) }}'
+    AND kcu.table_name = '{{ escape_relation_literal(relation.identifier|lower) }}' 
     AND kcu.constraint_name = (
       SELECT constraint_name
       FROM `{{ relation.database|lower }}`.information_schema.table_constraints
-      WHERE table_catalog = '{{ relation.database|lower }}'
-        AND table_schema = '{{ relation.schema|lower }}'
-        AND table_name = '{{ relation.identifier|lower }}' 
+      WHERE table_catalog = '{{ escape_relation_literal(relation.database|lower) }}'
+        AND table_schema = '{{ escape_relation_literal(relation.schema|lower) }}'
+        AND table_name = '{{ escape_relation_literal(relation.identifier|lower) }}' 
         AND constraint_type = 'PRIMARY KEY'
     )
   ORDER BY kcu.ordinal_position;
@@ -73,15 +73,15 @@
     AND rc.unique_constraint_schema = ukcu.constraint_schema
     AND rc.unique_constraint_name = ukcu.constraint_name
     AND kcu.ordinal_position = ukcu.ordinal_position
-  WHERE kcu.table_catalog = '{{ relation.database|lower }}'
-    AND kcu.table_schema = '{{ relation.schema|lower }}'
-    AND kcu.table_name = '{{ relation.identifier|lower }}'
+  WHERE kcu.table_catalog = '{{ escape_relation_literal(relation.database|lower) }}'
+    AND kcu.table_schema = '{{ escape_relation_literal(relation.schema|lower) }}'
+    AND kcu.table_name = '{{ escape_relation_literal(relation.identifier|lower) }}'
     AND kcu.constraint_name IN (
       SELECT constraint_name
       FROM `{{ relation.database|lower }}`.information_schema.table_constraints
-      WHERE table_catalog = '{{ relation.database|lower }}'
-        AND table_schema = '{{ relation.schema|lower }}'
-        AND table_name = '{{ relation.identifier|lower }}'
+      WHERE table_catalog = '{{ escape_relation_literal(relation.database|lower) }}'
+        AND table_schema = '{{ escape_relation_literal(relation.schema|lower) }}'
+        AND table_name = '{{ escape_relation_literal(relation.identifier|lower) }}'
         AND constraint_type = 'FOREIGN KEY'
     )
   ORDER BY kcu.ordinal_position;

dbt/include/databricks/macros/relations/components/row_filter.sql

@@ -46,9 +46,9 @@ The duplication exists because:
     filter_name,
     target_columns
   FROM `{{ relation.database }}`.`information_schema`.`row_filters`
-  WHERE table_catalog = '{{ relation.database | lower }}'
-    AND table_schema = '{{ relation.schema | lower }}'
-    AND table_name = '{{ relation.identifier | lower }}'
+  WHERE table_catalog = '{{ escape_relation_literal(relation.database | lower) }}'
+    AND table_schema = '{{ escape_relation_literal(relation.schema | lower) }}'
+    AND table_name = '{{ escape_relation_literal(relation.identifier | lower) }}'
 {%- endmacro -%}
 
 

dbt/include/databricks/macros/relations/escape.sql

@@ -0,0 +1,5 @@
+{% macro escape_relation_literal(value) -%}
+  {%- if value is not none -%}
+    {{- value | string | replace('\\', '\\\\') | replace("'", "\\'") -}}
+  {%- endif -%}
+{%- endmacro %}

dbt/include/databricks/macros/relations/tags.sql

@@ -11,9 +11,9 @@
 {% macro fetch_tags_sql(relation) -%}
   SELECT tag_name, tag_value
   FROM `system`.`information_schema`.`table_tags`
-  WHERE catalog_name = '{{ relation.database|lower }}' 
-    AND schema_name = '{{ relation.schema|lower }}'
-    AND table_name = '{{ relation.identifier|lower }}'
+  WHERE catalog_name = '{{ escape_relation_literal(relation.database|lower) }}' 
+    AND schema_name = '{{ escape_relation_literal(relation.schema|lower) }}'
+    AND table_name = '{{ escape_relation_literal(relation.identifier|lower) }}'
 {%- endmacro -%}
 
 {% macro apply_tags(relation, set_tags) -%}

tests/unit/macros/adapters/test_catalog_macros.py

@@ -128,6 +128,53 @@ def test_get_catalog_relations_where_clause_sql(self, template_bundle):
 
         self.assert_sql_equal(result, expected)
 
+    def test_get_catalog_schemas_where_clause_sql_escapes_special_characters(self, template_bundle):
+        catalog = "cat'a\\log"
+        schemas = [["cat'a\\log", "sch'e\\ma1"], ["cat'a\\log", "sch'e\\ma2"]]
+
+        result = self.run_macro(
+            template_bundle.template,
+            "databricks__get_catalog_schemas_where_clause_sql",
+            catalog,
+            schemas,
+        )
+
+        expected = """
+            where table_catalog = 'cat\\'a\\\\log'
+            and (
+                table_schema = 'sch\\'e\\\\ma1' or
+                table_schema = 'sch\\'e\\\\ma2'
+            )
+        """
+        self.assert_sql_equal(result, expected)
+
+    def test_get_catalog_relations_where_clause_sql_escapes_special_characters(
+        self, template_bundle
+    ):
+        relation1 = MagicMock()
+        relation1.schema = "sch'e\\ma1"
+        relation1.identifier = "orders'eu\\test1"
+
+        relation2 = MagicMock()
+        relation2.schema = "sch'e\\ma2"
+        relation2.identifier = "orders'eu\\test2"
+
+        result = self.run_macro(
+            template_bundle.template,
+            "databricks__get_catalog_relations_where_clause_sql",
+            "cat'a\\log",
+            [relation1, relation2],
+        )
+
+        expected = """
+            where table_catalog = 'cat\\'a\\\\log'
+            and (
+                (table_schema = 'sch\\'e\\\\ma1' and table_name = 'orders\\'eu\\\\test1') or
+                (table_schema = 'sch\\'e\\\\ma2' and table_name = 'orders\\'eu\\\\test2')
+            )
+        """
+        self.assert_sql_equal(result, expected)
+
     def test_get_catalog(self, template_bundle, context):
         """Test the full get_catalog macro with mocked run_query"""
         information_schema = {"database": "test_catalog"}

tests/unit/macros/adapters/test_metadata_macros.py

@@ -166,6 +166,48 @@ def test_get_uc_tables_sql_without_identifier(self, template_bundle, mock_schema
         """  # noqa
         self.assert_sql_equal(result, expected_sql)
 
+    def test_get_relation_last_modified_sql_escapes_special_characters(
+        self, context, template_bundle, mock_information_schema
+    ):
+        context["current_timestamp"] = Mock(return_value="current_timestamp()")
+        mock_information_schema.is_hive_metastore.return_value = False
+        mock_information_schema.database = "cat'a\\log"
+
+        rel1 = Mock()
+        rel1.schema = "sch'e\\ma1"
+        rel1.identifier = "orders'eu\\test1"
+
+        result = self.run_macro(
+            template_bundle.template,
+            "get_relation_last_modified_sql",
+            mock_information_schema,
+            [rel1],
+        )
+
+        assert "where table_catalog = 'cat\\'a\\\\log'" in result
+        assert "table_schema = 'sch\\'e\\\\ma1'" in result
+        assert "table_name = 'orders\\'eu\\\\test1'" in result
+
+    def test_get_view_description_sql_escapes_special_characters(self, template_bundle, relation):
+        relation.database = "cat'a\\log"
+        relation.schema = "sch'e\\ma"
+        relation.identifier = "orders'eu\\test"
+        result = self.run_macro(template_bundle.template, "get_view_description_sql", relation)
+
+        assert "where table_catalog = 'cat\\'a\\\\log'" in result
+        assert "and table_schema = 'sch\\'e\\\\ma'" in result
+        assert "and table_name = 'orders\\'eu\\\\test'" in result
+
+    def test_get_uc_tables_sql_escapes_special_characters(self, template_bundle, relation):
+        relation.database = "cat'a\\log"
+        relation.schema = "sch'e\\ma"
+        relation.identifier = "orders'eu\\test"
+        result = self.run_macro(template_bundle.template, "get_uc_tables_sql", relation)
+
+        assert "where table_catalog = 'cat\\'a\\\\log'" in result
+        assert "and table_schema = 'sch\\'e\\\\ma'" in result
+        assert "and table_name = 'orders\\'eu\\\\test'" in result
+
     def test_list_schemas_sql_with_database(self, template_bundle):
         result = self.run_macro(template_bundle.template, "list_schemas_sql", "my_catalog")
 

tests/unit/macros/base.py

@@ -74,6 +74,9 @@ def mock_relation_create(database=None, schema=None, identifier=None, type=None)
             "is_incremental": Mock(return_value=False),
             "api": mock_api,
             "local_md5": lambda s: f"hash({s})",
+            "escape_relation_literal": lambda val: (
+                str(val).replace("\\", "\\\\").replace("'", "\\'") if val is not None else ""
+            ),
         }
         return context
 

tests/unit/macros/relations/components/test_column_mask_macros.py

@@ -0,0 +1,38 @@
+import pytest
+
+from tests.unit.macros.base import MacroTestBase
+
+
+class TestColumnMaskMacros(MacroTestBase):
+    @pytest.fixture
+    def template_name(self) -> str:
+        return "column_mask.sql"
+
+    @pytest.fixture
+    def macro_folders_to_load(self) -> list:
+        return ["macros/relations/components", "macros/relations", "macros"]
+
+    def test_fetch_column_masks_sql(self, template_bundle):
+        sql = self.render_bundle(template_bundle, "fetch_column_masks_sql")
+        expected = self.clean_sql(
+            "SELECT column_name, mask_name, using_columns "
+            "FROM `system`.`information_schema`.`column_masks` "
+            "WHERE table_catalog = 'some_database' "
+            "AND table_schema = 'some_schema' "
+            "AND table_name = 'some_table';"
+        )
+        assert sql == expected
+
+    def test_fetch_column_masks_sql_escapes_special_characters(self, template_bundle):
+        template_bundle.relation.database = "cat'a\\log"
+        template_bundle.relation.schema = "sch'e\\ma"
+        template_bundle.relation.identifier = "orders'eu\\test"
+        sql = self.render_bundle(template_bundle, "fetch_column_masks_sql")
+        expected = self.clean_sql(
+            "SELECT column_name, mask_name, using_columns "
+            "FROM `system`.`information_schema`.`column_masks` "
+            "WHERE table_catalog = 'cat\\'a\\\\log' "
+            "AND table_schema = 'sch\\'e\\\\ma' "
+            "AND table_name = 'orders\\'eu\\\\test';"
+        )
+        assert sql == expected

tests/unit/macros/relations/components/test_column_tags_macros.py

@@ -93,3 +93,28 @@ def boom(name):
             passthrough_statement.template, "unset_column_tags", passthrough_statement.relation, []
         )
         assert "unset tags" not in sql
+
+    def test_fetch_column_tags_sql(self, template_bundle):
+        sql = self.render_bundle(template_bundle, "fetch_column_tags_sql")
+        expected = self.clean_sql(
+            "SELECT column_name, tag_name, tag_value "
+            "FROM `system`.`information_schema`.`column_tags` "
+            "WHERE catalog_name = 'some_database' "
+            "AND schema_name = 'some_schema' "
+            "AND table_name = 'some_table';"
+        )
+        assert sql == expected
+
+    def test_fetch_column_tags_sql_escapes_special_characters(self, template_bundle):
+        template_bundle.relation.database = "cat'a\\log"
+        template_bundle.relation.schema = "sch'e\\ma"
+        template_bundle.relation.identifier = "orders'eu\\test"
+        sql = self.render_bundle(template_bundle, "fetch_column_tags_sql")
+        expected = self.clean_sql(
+            "SELECT column_name, tag_name, tag_value "
+            "FROM `system`.`information_schema`.`column_tags` "
+            "WHERE catalog_name = 'cat\\'a\\\\log' "
+            "AND schema_name = 'sch\\'e\\\\ma' "
+            "AND table_name = 'orders\\'eu\\\\test';"
+        )
+        assert sql == expected