From f09dcb09b55a348e1c2b5bb9b327c858f2f58208 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 27 Feb 2024 21:45:02 -0800 Subject: [PATCH 01/22] [Views] Implement Materialized Views; Integrate with Spark SQL --- .../iceberg/view/ViewVersionReplace.java | 9 + .../sql/catalyst/analysis/CheckViews.scala | 1 + .../sql/catalyst/analysis/ResolveViews.scala | 1 + .../analysis/RewriteViewCommands.scala | 6 +- .../IcebergSparkSqlExtensionsParser.scala | 18 +- .../logical/views/CreateIcebergView.scala | 5 +- .../v2/CreateMaterializedViewExec.scala | 160 +++++++++++++++ .../v2/CreateOrReplaceTagExec.scala | 3 +- .../datasources/v2/DropV2ViewExec.scala | 38 ++++ .../v2/ExtendedDataSourceV2Strategy.scala | 40 ++-- .../extensions/TestMaterializedViews.java | 185 ++++++++++++++++++ .../iceberg/spark/MaterializedViewUtil.java | 91 +++++++++ .../apache/iceberg/spark/SparkCatalog.java | 97 ++++++++- .../spark/source/SparkMaterializedView.java | 57 ++++++ 14 files changed, 691 insertions(+), 20 deletions(-) create mode 100644 spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala create mode 100644 spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java create mode 100644 spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java create mode 100644 spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java diff --git a/core/src/main/java/org/apache/iceberg/view/ViewVersionReplace.java b/core/src/main/java/org/apache/iceberg/view/ViewVersionReplace.java index 8b3d087940a5..0150fd2a2a44 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewVersionReplace.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewVersionReplace.java @@ -28,6 +28,7 @@ import static org.apache.iceberg.TableProperties.COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT; import java.util.List; +import java.util.Optional; import org.apache.iceberg.EnvironmentContext; import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.Namespace; @@ -56,6 +57,14 @@ public ViewVersion apply() { } ViewMetadata internalApply() { + // Replacing a materialized view is not supported because the old storage location will wrongly + // transfer to the new version + // if not handled properly. + Preconditions.checkState( + Optional.ofNullable(base.properties().get("iceberg.materialized.view")) + .orElse("false") + .equals("false"), + "Cannot replace a materialized view with a new version"); Preconditions.checkState( !representations.isEmpty(), "Cannot replace view without specifying a query"); Preconditions.checkState(null != schema, "Cannot replace view without specifying schema"); diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala index 319ab78a5348..6f6b41ad11c1 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala @@ -49,6 +49,7 @@ object CheckViews extends (LogicalPlan => Unit) { _, replace, _, + _, _) => verifyColumnCount(resolvedIdent, columnAliases, query) SchemaUtils.checkColumnNameDuplication( diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala index b1ebd6cb1266..4f304467e31f 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala @@ -76,6 +76,7 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look _, _, _, + _, _) if query.resolved && !c.rewritten => val aliased = aliasColumns(query, columnAliases, columnComments) c.copy( diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala index c47b7d6ef6ac..3dbe02149bcd 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala @@ -41,7 +41,8 @@ import scala.collection.mutable * ResolveSessionCatalog exits early for some v2 View commands, * thus they are pre-substituted here and then handled in ResolveViews */ -case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog { +case class RewriteViewCommands(spark: SparkSession, materialized: Boolean) + extends Rule[LogicalPlan] with LookupCatalog { import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ @@ -71,7 +72,8 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi comment = comment, properties = properties, allowExisting = allowExisting, - replace = replace) + replace = replace, + materialized = materialized) case view @ ShowViews(UnresolvedNamespace(Seq()), pattern, output) => if (ViewUtil.isViewCatalog(catalogManager.currentCatalog)) { diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala index b25333d56787..ada7b16533ce 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala @@ -118,8 +118,12 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) if (isIcebergCommand(sqlTextAfterSubstitution)) { parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) } .asInstanceOf[LogicalPlan] + } else if (isCreateMaterializedView(sqlText)) { + RewriteViewCommands(SparkSession.active, true).apply( + delegate.parsePlan(replaceCreateMaterializedViewWithCreateView(sqlText)) + ) } else { - RewriteViewCommands(SparkSession.active).apply(delegate.parsePlan(sqlText)) + RewriteViewCommands(SparkSession.active, false).apply(delegate.parsePlan(sqlText)) } } @@ -157,6 +161,18 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) SparkProcedures.names().asScala.map("system." + _).exists(normalized.contains) } + private def isCreateMaterializedView(sqlText: String): Boolean = { + sqlText.toLowerCase.contains("create materialized view") + } + + def replaceCreateMaterializedViewWithCreateView(input: String): String = { + // Regex pattern to match "create materialized view" in a case-insensitive manner + val pattern = "(?i)create materialized view".r + + // Replace all occurrences of the pattern with "create view" + pattern.replaceAllIn(input, "create view") + } + private def isSnapshotRefDdl(normalized: String): Boolean = { normalized.contains("create branch") || normalized.contains("replace branch") || diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala index 84a00a4a9a88..9f46ab0f0646 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala @@ -22,8 +22,8 @@ import org.apache.spark.sql.catalyst.analysis.AnalysisContext import org.apache.spark.sql.catalyst.plans.logical.AnalysisOnlyCommand import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -// Align Iceberg's CreateIcebergView with Spark’s CreateViewCommand by extending AnalysisOnlyCommand. -// The command’s children are analyzed then hidden, so the optimizer/planner won’t traverse the view body. +// Align Iceberg's CreateIcebergView with Spark's CreateViewCommand by extending AnalysisOnlyCommand. +// The command's children are analyzed then hidden, so the optimizer/planner won't traverse the view body. case class CreateIcebergView( child: LogicalPlan, queryText: String, @@ -36,6 +36,7 @@ case class CreateIcebergView( allowExisting: Boolean, replace: Boolean, rewritten: Boolean = false, + materialized: Boolean = false, isAnalyzed: Boolean = false) extends AnalysisOnlyCommand { diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala new file mode 100644 index 000000000000..265ed95e33ae --- /dev/null +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.execution.datasources.v2 + +import java.util.UUID +import org.apache.hadoop.conf.Configuration +import org.apache.iceberg +import org.apache.iceberg.FileFormat +import org.apache.iceberg.PartitionSpec +import org.apache.iceberg.hadoop.HadoopTables +import org.apache.iceberg.relocated.com.google.common.base.Preconditions +import org.apache.iceberg.spark.MaterializedViewUtil +import org.apache.iceberg.spark.SparkSchemaUtil +import org.apache.iceberg.spark.SparkWriteOptions +import org.apache.iceberg.spark.source.SparkTable +import org.apache.spark.sql.SaveMode +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.Table +import org.apache.spark.sql.connector.catalog.ViewCatalog +import org.apache.spark.sql.types.StructType +import scala.collection.JavaConverters._ + +case class CreateMaterializedViewExec( + catalog: ViewCatalog, + ident: Identifier, + queryText: String, + viewSchema: StructType, + columnAliases: Seq[String], + columnComments: Seq[Option[String]], + queryColumnNames: Seq[String], + comment: Option[String], + properties: Map[String, String], + allowExisting: Boolean, + replace: Boolean) extends LeafV2CommandExec { + + override def output: Seq[Attribute] = Nil + + override protected def run(): Seq[InternalRow] = { + + val viewLocation = properties.get("location") + Preconditions.checkArgument(viewLocation.isDefined) + + val storageTableLocation = viewLocation + "/storage/v1" + + // Create the storage table in the Hadoop catalog so it is explicitly registered in the Spark catalog + val tables: HadoopTables = new HadoopTables(new Configuration()) + val icebergSchema = SparkSchemaUtil.convert(viewSchema) + // TODO: Add support for partitioning the storage table + val spec: PartitionSpec = PartitionSpec.builderFor(icebergSchema).build + + val table: iceberg.Table = tables.create(icebergSchema, spec, storageTableLocation) + + val baseTables = MaterializedViewUtil.extractBaseTables(queryText).asScala.toList + val baseTableSnapshots = getBaseTableSnapshots(baseTables) + val baseTableSnapshotsProperties = baseTableSnapshots.map{ + case (key, value) => ( + MaterializedViewUtil.MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX + key.toString + ) -> value.toString + } + + session.sql(queryText).write.format("iceberg").option( + SparkWriteOptions.WRITE_FORMAT, FileFormat.PARQUET.toString + ).mode(SaveMode.Append).save(storageTableLocation) + + val updateProperties = table.updateProperties() + baseTableSnapshotsProperties.foreach { + case (key, value) => updateProperties.set(key, value) + } + updateProperties.commit() + + table.refresh() + + createMaterializedView(storageTableLocation) + Nil + } + + override def simpleString(maxFields: Int): String = { + s"CreateMaterializedViewExec: ${ident}" + } + + private def createMaterializedView(storageTableLocation: String): Unit = { + val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name + val currentCatalog = if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null + val currentNamespace = session.sessionState.catalogManager.currentNamespace + + val engineVersion = "Spark " + org.apache.spark.SPARK_VERSION + val newProperties = properties ++ + comment.map(ViewCatalog.PROP_COMMENT -> _) + + (ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion, + ViewCatalog.PROP_ENGINE_VERSION -> engineVersion) + + (MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY -> "true") + + (MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY -> storageTableLocation) + + if (replace) { + // CREATE OR REPLACE VIEW + if (catalog.viewExists(ident)) { + catalog.dropView(ident) + } + // FIXME: replaceView API doesn't exist in Spark 3.5 + catalog.createView( + ident, + queryText, + currentCatalog, + currentNamespace, + viewSchema, + queryColumnNames.toArray, + columnAliases.toArray, + columnComments.map(c => c.orNull).toArray, + newProperties.asJava) + } else { + try { + // CREATE VIEW [IF NOT EXISTS] + catalog.createView( + ident, + queryText, + currentCatalog, + currentNamespace, + viewSchema, + queryColumnNames.toArray, + columnAliases.toArray, + columnComments.map(c => c.orNull).toArray, + newProperties.asJava) + } catch { + // TODO: Make sure the existing view is also a materialized view + case _: ViewAlreadyExistsException if allowExisting => // Ignore + } + } + } + + private def getBaseTableSnapshots(baseTables: List[Table]): Map[UUID, Long] = { + baseTables.map { + case sparkTable: SparkTable => + val snapshot = Option(sparkTable.table().currentSnapshot()) + val snapshotId = snapshot.map(_.snapshotId().longValue()).getOrElse(0L) + (sparkTable.table().uuid(), snapshotId) + case _ => + throw new UnsupportedOperationException("Only Spark tables are supported") + }.toMap + } +} diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala index e486892614cb..03c7e1385fff 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala @@ -40,7 +40,8 @@ case class CreateOrReplaceTagExec( override lazy val output: Seq[Attribute] = Nil override protected def run(): Seq[InternalRow] = { - catalog.loadTable(ident) match { + catalog + .loadTable(ident) match { case iceberg: SparkTable => val snapshotId: java.lang.Long = tagOptions.snapshotId .orElse(Option(iceberg.table.currentSnapshot()).map(_.snapshotId())) diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala index 6dd1188b78e8..551e802b6bf8 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala @@ -18,6 +18,14 @@ */ package org.apache.spark.sql.execution.datasources.v2 +import org.apache.hadoop.conf.Configuration +import org.apache.iceberg.catalog.Namespace +import org.apache.iceberg.catalog.TableIdentifier +import org.apache.iceberg.exceptions +import org.apache.iceberg.hadoop.HadoopTables +import org.apache.iceberg.spark.MaterializedViewUtil +import org.apache.iceberg.spark.SparkCatalog +import org.apache.iceberg.view.View import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.NoSuchViewException import org.apache.spark.sql.catalyst.expressions.Attribute @@ -30,6 +38,36 @@ case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boo override lazy val output: Seq[Attribute] = Nil override protected def run(): Seq[InternalRow] = { + val icebergCatalog = catalog.asInstanceOf[SparkCatalog].icebergCatalog() + val icebergViewCatalog = icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] + var view: Option[View] = None + try { + view = Some(icebergViewCatalog.loadView(TableIdentifier.of(Namespace.of(ident.namespace(): _*), ident.name()))) + } catch { + case e: exceptions.NoSuchViewException => { + if (!ifExists) { + throw new NoSuchViewException(ident) + } + } + } + // if view is not null read the properties and check if it is a materialized view + view match { + case Some(v) => { + val viewProperties = v.properties(); + if (Option( + viewProperties.get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY + )).getOrElse("false").equals("true")) { + // get the storage table location then drop the storage table + val storageTableLocation = viewProperties.get( + MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY + ) + val tables: HadoopTables = new HadoopTables(new Configuration()) + tables.dropTable(storageTableLocation) + } + } + case _ => + } + val dropped = catalog.dropView(ident) if (!dropped && !ifExists) { throw new NoSuchViewException(ident) diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala index 6b340b72496e..2549bd61bf8e 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala @@ -149,19 +149,35 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi allowExisting, replace, _, + materialized, _) => - CreateV2ViewExec( - catalog = viewCatalog, - ident = ident, - queryText = queryText, - columnAliases = columnAliases, - columnComments = columnComments, - queryColumnNames = queryColumnNames, - viewSchema = query.schema, - comment = comment, - properties = properties, - allowExisting = allowExisting, - replace = replace) :: Nil + if (materialized) { + CreateMaterializedViewExec( + catalog = viewCatalog, + ident = ident, + queryText = queryText, + columnAliases = columnAliases, + columnComments = columnComments, + queryColumnNames = queryColumnNames, + viewSchema = query.schema, + comment = comment, + properties = properties, + allowExisting = allowExisting, + replace = replace) :: Nil + } else { + CreateV2ViewExec( + catalog = viewCatalog, + ident = ident, + queryText = queryText, + columnAliases = columnAliases, + columnComments = columnComments, + queryColumnNames = queryColumnNames, + viewSchema = query.schema, + comment = comment, + properties = properties, + allowExisting = allowExisting, + replace = replace) :: Nil + } case DescribeRelation(ResolvedV2View(catalog, ident), _, isExtended, output) => DescribeV2ViewExec(output, catalog.loadView(ident), isExtended) :: Nil diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java new file mode 100644 index 000000000000..99e0cefd7048 --- /dev/null +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.extensions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Map; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.spark.MaterializedViewUtil; +import org.apache.iceberg.spark.Spark3Util; +import org.apache.iceberg.spark.SparkCatalogConfig; +import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; +import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; +import org.apache.spark.sql.connector.catalog.CatalogPlugin; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.connector.catalog.ViewCatalog; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runners.Parameterized; + +public class TestMaterializedViews extends SparkExtensionsTestBase { + private static final Namespace NAMESPACE = Namespace.of("default"); + private final String tableName = "table"; + private final String materializedViewName = "materialized_view"; + + @Before + public void before() { + spark.conf().set("spark.sql.defaultCatalog", catalogName); + sql("USE %s", catalogName); + sql("CREATE NAMESPACE IF NOT EXISTS %s", NAMESPACE); + sql("CREATE TABLE %s (id INT, data STRING)", tableName); + } + + @After + public void removeTable() { + sql("USE %s", catalogName); + sql("DROP VIEW IF EXISTS %s", materializedViewName); + sql("DROP TABLE IF EXISTS %s", tableName); + } + + @Parameterized.Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") + public static Object[][] parameters() { + return new Object[][] { + { + SparkCatalogConfig.SPARK_WITH_VIEWS.catalogName(), + SparkCatalogConfig.SPARK_WITH_VIEWS.implementation(), + SparkCatalogConfig.SPARK_WITH_VIEWS.properties() + } + }; + } + + public TestMaterializedViews( + String catalog, String implementation, Map properties) { + super(catalog, implementation, properties); + } + + @Test + public void assertReadFromStorageTableWhenFresh() throws IOException { + File location = Files.createTempDirectory("materialized-view-test").toFile(); + sql("DROP VIEW IF EXISTS %s", materializedViewName); + sql( + "CREATE MATERIALIZED VIEW %s TBLPROPERTIES ('location' = '%s') AS SELECT id, data FROM %s", + materializedViewName, location.getAbsolutePath(), tableName); + + // Assert that number of records in the materialized view is the same as the number of records + // in the table + assertThat(sql("SELECT * FROM %s", materializedViewName).size()) + .isEqualTo(sql("SELECT * FROM %s", tableName).size()); + + // Assert that the catalog loadView method returns NoSuchViewException because the view is fresh + assertThatThrownBy( + () -> + sparkViewCatalog() + .loadView(Identifier.of(new String[] {"default"}, materializedViewName))) + .isInstanceOf(NoSuchViewException.class); + + // Assert that the catalog loadTable method returns the materialized view storage table + try { + assertThat( + sparkTableCatalog() + .loadTable(Identifier.of(new String[] {"default"}, materializedViewName)) + .name()) + .isEqualTo( + icebergViewCatalog() + .loadView(TableIdentifier.of("default", materializedViewName)) + .properties() + .get(MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY)); + } catch (NoSuchTableException e) { + fail("Materialized view storage table not found"); + } + } + + @Test + public void assertNotReadFromStorageTableWhenStale() throws IOException { + File location = Files.createTempDirectory("materialized-view-test").toFile(); + sql( + "CREATE MATERIALIZED VIEW %s TBLPROPERTIES ('location' = '%s') AS SELECT id, data FROM %s", + materializedViewName, location.getAbsolutePath(), tableName); + + // Insert one row to the table so the materialized view becomes stale + sql("INSERT INTO %s VALUES (1, 'a')", tableName); + + // Assert that number of records in the materialized view is the same as the number of records + // in the table + assertThat(sql("SELECT * FROM %s", materializedViewName).size()) + .isEqualTo(sql("SELECT * FROM %s", tableName).size()); + + // Assert that the catalog loadView method returns the view object + try { + assertThat( + sparkViewCatalog() + .loadView(Identifier.of(new String[] {"default"}, materializedViewName)) + .name()) + .isEqualTo( + icebergViewCatalog() + .loadView(TableIdentifier.of("default", materializedViewName)) + .name()); + } catch (NoSuchViewException e) { + fail("Materialized view not found"); + } + + // Assert that the catalog loadTable fails with NoSuchTableException because the view is stale + assertThatThrownBy( + () -> + sparkTableCatalog() + .loadTable(Identifier.of(new String[] {"default"}, materializedViewName))) + .isInstanceOf(NoSuchTableException.class); + } + + @Test + public void assertShowTablesDoesNotShowStorageTable() throws IOException { + File location = Files.createTempDirectory("materialized-view-test").toFile(); + sql( + "CREATE MATERIALIZED VIEW %s TBLPROPERTIES ('location' = '%s') AS SELECT id, data FROM %s", + materializedViewName, location.getAbsolutePath(), tableName); + + // Assert that the storage table is not shown in the list of tables + assertThat(sql("SHOW TABLES").size() == 2); + } + + private ViewCatalog sparkViewCatalog() { + CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); + return (ViewCatalog) catalogPlugin; + } + + private TableCatalog sparkTableCatalog() { + CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); + return (TableCatalog) catalogPlugin; + } + + private org.apache.iceberg.catalog.ViewCatalog icebergViewCatalog() { + Catalog icebergCatalog = Spark3Util.loadIcebergCatalog(spark, catalogName); + assertThat(icebergCatalog).isInstanceOf(org.apache.iceberg.catalog.ViewCatalog.class); + return (org.apache.iceberg.catalog.ViewCatalog) icebergCatalog; + } + + // TODO Add DROP MATERIALIZED VIEW test + // TODO Assert materialized view creation fails when the location is not provided + // TODO Test cannot replace a materialized view with a new version +} diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java new file mode 100644 index 000000000000..f6dc101fe4fb --- /dev/null +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation; +import org.apache.spark.sql.catalyst.parser.ParseException; +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; +import org.apache.spark.sql.connector.catalog.Table; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import scala.collection.JavaConverters; + +// Possible to merge with Spark3Util +public class MaterializedViewUtil { + + private MaterializedViewUtil() {} + + public static final String MATERIALIZED_VIEW_PROPERTY_KEY = "iceberg.materialized.view"; + public static final String MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY = + "iceberg.materialized.view.storage.location"; + public static final String MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX = "base.snapshot."; + + public static List extractBaseTables(String query) { + return extractBaseTableIdentifiers(query).stream() + .filter(optional -> !optional.isEmpty()) + .map(id -> toSparkTable(id).get()) + .collect(Collectors.toList()); + } + + private static List> extractBaseTableIdentifiers(String query) { + try { + // Parse the SQL query to get the LogicalPlan + LogicalPlan logicalPlan = SparkSession.active().sessionState().sqlParser().parsePlan(query); + + // Recursively traverse the LogicalPlan to extract base table names + return extractBaseTableIdentifiers(logicalPlan).stream() + .distinct() + .collect(Collectors.toList()); + } catch (ParseException e) { + throw new IllegalArgumentException("Failed to parse the SQL query: " + query, e); + } + } + + private static List> extractBaseTableIdentifiers(LogicalPlan plan) { + if (plan instanceof UnresolvedRelation) { + UnresolvedRelation relation = (UnresolvedRelation) plan; + List> result = Lists.newArrayListWithCapacity(1); + result.add(JavaConverters.seqAsJavaList(relation.multipartIdentifier())); + return result; + } else { + return (JavaConverters.seqAsJavaList(plan.children())) + .stream() + .flatMap(child -> extractBaseTableIdentifiers(child).stream()) + .collect(Collectors.toList()); + } + } + + public static Optional
toSparkTable(List multipartIdent) { + Spark3Util.CatalogAndIdentifier catalogAndIdentifier = + Spark3Util.catalogAndIdentifier(SparkSession.active(), multipartIdent); + if (catalogAndIdentifier.catalog() instanceof TableCatalog) { + TableCatalog tableCatalog = (TableCatalog) catalogAndIdentifier.catalog(); + try { + return Optional.of(tableCatalog.loadTable(catalogAndIdentifier.identifier())); + } catch (Exception e) { + return Optional.empty(); + } + } + return Optional.empty(); + } +} diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index 31e6874c6739..a71323adaef8 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CachingCatalog; @@ -63,6 +64,7 @@ import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.spark.actions.SparkActions; import org.apache.iceberg.spark.source.SparkChangelogTable; +import org.apache.iceberg.spark.source.SparkMaterializedView; import org.apache.iceberg.spark.source.SparkTable; import org.apache.iceberg.spark.source.SparkView; import org.apache.iceberg.spark.source.StagedSparkTable; @@ -593,7 +595,20 @@ public View loadView(Identifier ident) throws NoSuchViewException { if (null != asViewCatalog) { try { org.apache.iceberg.view.View view = asViewCatalog.loadView(buildIdentifier(ident)); - return new SparkView(catalogName, view); + // Check if the view is a materialized view. If it is, and storage table is fresh, return + // NoSuchViewException so + // loadTable is attempted instead. + if (view.properties() + .get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY) + .equals("true")) { + if (isFresh(view)) { + throw new NoSuchViewException(ident); + } else { + return new SparkView(catalogName, view); + } + } else { + return new SparkView(catalogName, view); + } } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { throw new NoSuchViewException(ident); } @@ -602,6 +617,48 @@ public View loadView(Identifier ident) throws NoSuchViewException { throw new NoSuchViewException(ident); } + private boolean isFresh(org.apache.iceberg.view.View view) { + Preconditions.checkState( + view.properties().get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY).equals("true"), + "Cannot check freshness of non-materialized view."); + String storageTableLocation = + view.properties().get(MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY); + try { + Table storageTable = loadTable(new PathIdentifier(storageTableLocation)); + Map baseTableSnapshotsProperties = + storageTable.properties().entrySet().stream() + .filter( + entry -> + entry + .getKey() + .startsWith( + MaterializedViewUtil + .MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + List
baseTables = MaterializedViewUtil.extractBaseTables(view.sqlFor("spark").sql()); + + for (Table baseTable : baseTables) { + org.apache.iceberg.Table icebergBaseTable = ((SparkTable) baseTable).table(); + String snapshotId = + String.valueOf( + icebergBaseTable.currentSnapshot() == null + ? 0 + : icebergBaseTable.currentSnapshot().snapshotId()); + if (!baseTableSnapshotsProperties + .get( + MaterializedViewUtil.MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX + + icebergBaseTable.uuid()) + .equals(snapshotId)) { + return false; + } + } + return true; + } catch (NoSuchTableException e) { + throw new IllegalStateException( + "Could not load materialized view storage table from catalog.", e); + } + } + @Override public View createView( Identifier ident, @@ -634,7 +691,19 @@ public View createView( .withLocation(properties.get("location")) .withProperties(props) .create(); - return new SparkView(catalogName, view); + if (props.get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY).equals("true")) { + String storageTableLocation = + properties.get(MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY); + try { + Table storageTable = loadTable(new PathIdentifier(storageTableLocation)); + return new SparkMaterializedView(catalogName, view, storageTable); + } catch (NoSuchTableException e) { + throw new IllegalStateException( + "Could not load materialized view storage table from catalog.", e); + } + } else { + return new SparkView(catalogName, view); + } } catch (org.apache.iceberg.exceptions.NoSuchNamespaceException e) { throw new NoSuchNamespaceException(currentNamespace); } catch (AlreadyExistsException e) { @@ -888,11 +957,35 @@ private static void checkNotPathIdentifier(Identifier identifier, String method) } } + // TODO Remove @SuppressWarnings + @SuppressWarnings("checkstyle:CyclomaticComplexity") private Table load(Identifier ident) { if (isPathIdentifier(ident)) { return loadFromPathIdentifier((PathIdentifier) ident); } + // Check if materialized view. If fresh, return the SparkMaterializedView. + if (null != asViewCatalog) { + try { + org.apache.iceberg.view.View view = asViewCatalog.loadView(buildIdentifier(ident)); + if (view.properties() + .get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY) + .equals("true")) { + if (isFresh(view)) { + String storageTableLocation = + view.properties() + .get(MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY); + return new SparkMaterializedView( + catalogName, + view, + loadFromPathIdentifier(new PathIdentifier(storageTableLocation))); + } + } + } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { + // Ignore. Just process as a normal table. + } + } + try { org.apache.iceberg.Table table = icebergCatalog.loadTable(buildIdentifier(ident)); return new SparkTable(table, !cacheEnabled); diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java new file mode 100644 index 000000000000..0c01ae449292 --- /dev/null +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.source; + +import java.util.Set; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.view.View; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.connector.catalog.SupportsRead; +import org.apache.spark.sql.connector.catalog.Table; +import org.apache.spark.sql.connector.catalog.TableCapability; +import org.apache.spark.sql.connector.read.ScanBuilder; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +public class SparkMaterializedView extends SparkView implements SupportsRead { + private final Table storageTable; + private SparkSession lazySpark; + + public SparkMaterializedView(String catalogName, View icebergView, Table storageTable) { + super(catalogName, icebergView); + this.storageTable = storageTable; + } + + @Override + public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) { + return ((SupportsRead) storageTable).newScanBuilder(options); + } + + private SparkSession sparkSession() { + if (lazySpark == null) { + this.lazySpark = SparkSession.active(); + } + + return lazySpark; + } + + @Override + public Set capabilities() { + return ImmutableSet.of(TableCapability.BATCH_READ); + } +} From 9b3faa7bb71fcc252f37514635cf32895caf07b1 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Sun, 10 Mar 2024 17:51:14 -0700 Subject: [PATCH 02/22] Represent the storage table using its catalog identifier --- .../iceberg/inmemory/InMemoryCatalog.java | 10 +- .../analysis/RewriteViewCommands.scala | 6 +- .../IcebergSparkSqlExtensionsParser.scala | 22 ++- .../logical/views/CreateIcebergView.scala | 3 +- .../v2/CreateMaterializedViewExec.scala | 70 ++++---- .../datasources/v2/DropV2ViewExec.scala | 12 +- .../v2/ExtendedDataSourceV2Strategy.scala | 69 ++++---- .../extensions/TestMaterializedViews.java | 155 ++++++++++++------ .../iceberg/spark/MaterializedViewUtil.java | 16 +- .../apache/iceberg/spark/SparkCatalog.java | 139 ++++++++-------- .../iceberg/spark/SparkCatalogConfig.java | 6 +- 11 files changed, 303 insertions(+), 205 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java index 80297310e82f..74730eb37e25 100644 --- a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java +++ b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java @@ -106,6 +106,7 @@ public void initialize(String name, Map properties) { closeableGroup.setSuppressCloseFailure(true); } + // protected for testing @Override protected TableOperations newTableOps(TableIdentifier tableIdentifier) { return new InMemoryTableOperations(io, tableIdentifier); @@ -354,6 +355,7 @@ public List listViews(Namespace namespace) { .collect(Collectors.toList()); } + // protected for testing @Override protected ViewOperations newViewOps(TableIdentifier identifier) { return new InMemoryViewOperations(io, identifier); @@ -401,12 +403,12 @@ protected Map properties() { return catalogProperties == null ? ImmutableMap.of() : catalogProperties; } - private class InMemoryTableOperations extends BaseMetastoreTableOperations { + public class InMemoryTableOperations extends BaseMetastoreTableOperations { private final FileIO fileIO; private final TableIdentifier tableIdentifier; private final String fullTableName; - InMemoryTableOperations(FileIO fileIO, TableIdentifier tableIdentifier) { + public InMemoryTableOperations(FileIO fileIO, TableIdentifier tableIdentifier) { this.fileIO = fileIO; this.tableIdentifier = tableIdentifier; this.fullTableName = fullTableName(catalogName, tableIdentifier); @@ -472,12 +474,12 @@ protected String tableName() { } } - private class InMemoryViewOperations extends BaseViewOperations { + public class InMemoryViewOperations extends BaseViewOperations { private final FileIO io; private final TableIdentifier identifier; private final String fullViewName; - InMemoryViewOperations(FileIO io, TableIdentifier identifier) { + public InMemoryViewOperations(FileIO io, TableIdentifier identifier) { this.io = io; this.identifier = identifier; this.fullViewName = ViewUtil.fullViewName(catalogName, identifier); diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala index 3dbe02149bcd..b75b93a3fa17 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala @@ -41,7 +41,7 @@ import scala.collection.mutable * ResolveSessionCatalog exits early for some v2 View commands, * thus they are pre-substituted here and then handled in ResolveViews */ -case class RewriteViewCommands(spark: SparkSession, materialized: Boolean) +case class RewriteViewCommands(spark: SparkSession, materializedViewOptions: Option[MaterializedViewOptions]) extends Rule[LogicalPlan] with LookupCatalog { import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ @@ -73,7 +73,7 @@ case class RewriteViewCommands(spark: SparkSession, materialized: Boolean) properties = properties, allowExisting = allowExisting, replace = replace, - materialized = materialized) + materializedViewOptions = materializedViewOptions) case view @ ShowViews(UnresolvedNamespace(Seq()), pattern, output) => if (ViewUtil.isViewCatalog(catalogManager.currentCatalog)) { @@ -209,3 +209,5 @@ case class RewriteViewCommands(spark: SparkSession, materialized: Boolean) tempFunctions.toSeq } } + +case class MaterializedViewOptions(storageTableIdentifier: Option[String]) diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala index ada7b16533ce..86306e76827e 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.MaterializedViewOptions import org.apache.spark.sql.catalyst.analysis.RewriteViewCommands import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.ParserInterface @@ -53,6 +54,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) private lazy val substitutor = substitutorCtor.newInstance(SQLConf.get) private lazy val astBuilder = new IcebergSqlExtensionsAstBuilder(delegate) + private lazy final val CREATE_MATERIALIZED_VIEW_PATTERN = "(?i)(CREATE)\\s+MATERIALIZED\\s+(VIEW)".r + private lazy final val MATERIALIZED_VIEW_STORED_AS_PATTERN = "(?i)STORED AS\\s*'(\\w+)'\\s*".r /** * Parse a string to a DataType. @@ -119,11 +122,11 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) } .asInstanceOf[LogicalPlan] } else if (isCreateMaterializedView(sqlText)) { - RewriteViewCommands(SparkSession.active, true).apply( - delegate.parsePlan(replaceCreateMaterializedViewWithCreateView(sqlText)) + RewriteViewCommands(SparkSession.active, Option(getMaterializedViewOptions(sqlText))).apply( + delegate.parsePlan(getCreateMaterializedViewStatement(sqlText)) ) } else { - RewriteViewCommands(SparkSession.active, false).apply(delegate.parsePlan(sqlText)) + RewriteViewCommands(SparkSession.active, None).apply(delegate.parsePlan(sqlText)) } } @@ -165,12 +168,15 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) sqlText.toLowerCase.contains("create materialized view") } - def replaceCreateMaterializedViewWithCreateView(input: String): String = { - // Regex pattern to match "create materialized view" in a case-insensitive manner - val pattern = "(?i)create materialized view".r + private def getCreateMaterializedViewStatement(sqlText: String): String = { + val replace1 = CREATE_MATERIALIZED_VIEW_PATTERN.replaceAllIn(sqlText, m => m.group(1) + " " + m.group(2)) + MATERIALIZED_VIEW_STORED_AS_PATTERN.replaceAllIn(replace1, "") + } - // Replace all occurrences of the pattern with "create view" - pattern.replaceAllIn(input, "create view") + private def getMaterializedViewOptions(sqlText: String): MaterializedViewOptions = { + val storedAsPattern = "(?i)STORED AS\\s*'(\\w+)'\\s*".r + val storageTableIdentifier = storedAsPattern.findFirstMatchIn(sqlText).map(_.group(1)) + MaterializedViewOptions(storageTableIdentifier) } private def isSnapshotRefDdl(normalized: String): Boolean = { diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala index 9f46ab0f0646..c21f730add94 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.plans.logical.views import org.apache.spark.sql.catalyst.analysis.AnalysisContext +import org.apache.spark.sql.catalyst.analysis.MaterializedViewOptions import org.apache.spark.sql.catalyst.plans.logical.AnalysisOnlyCommand import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -36,7 +37,7 @@ case class CreateIcebergView( allowExisting: Boolean, replace: Boolean, rewritten: Boolean = false, - materialized: Boolean = false, + materializedViewOptions: Option[MaterializedViewOptions] = None, isAnalyzed: Boolean = false) extends AnalysisOnlyCommand { diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala index 265ed95e33ae..bcff93dd1ab7 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -19,24 +19,22 @@ package org.apache.spark.sql.execution.datasources.v2 + import java.util.UUID -import org.apache.hadoop.conf.Configuration -import org.apache.iceberg -import org.apache.iceberg.FileFormat -import org.apache.iceberg.PartitionSpec -import org.apache.iceberg.hadoop.HadoopTables import org.apache.iceberg.relocated.com.google.common.base.Preconditions +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap import org.apache.iceberg.spark.MaterializedViewUtil -import org.apache.iceberg.spark.SparkSchemaUtil -import org.apache.iceberg.spark.SparkWriteOptions +import org.apache.iceberg.spark.Spark3Util +import org.apache.iceberg.spark.SparkCatalog import org.apache.iceberg.spark.source.SparkTable -import org.apache.spark.sql.SaveMode import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.connector.catalog.Table +import org.apache.spark.sql.connector.catalog.TableChange import org.apache.spark.sql.connector.catalog.ViewCatalog +import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.types.StructType import scala.collection.JavaConverters._ @@ -51,25 +49,36 @@ case class CreateMaterializedViewExec( comment: Option[String], properties: Map[String, String], allowExisting: Boolean, - replace: Boolean) extends LeafV2CommandExec { + replace: Boolean, + storageTableIdentifier: Option[String]) extends LeafV2CommandExec { override def output: Seq[Attribute] = Nil override protected def run(): Seq[InternalRow] = { - val viewLocation = properties.get("location") - Preconditions.checkArgument(viewLocation.isDefined) - - val storageTableLocation = viewLocation + "/storage/v1" + // Check if storageTableIdentifier is provided, if not, generate a default identifier + val sparkStorageTableIdentifier = storageTableIdentifier match { + case Some(identifier) => { + val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(session, identifier) + val storageTableCatalogName = catalogAndIdentifier.catalog().name() + Preconditions.checkState( + storageTableCatalogName.equals(catalog.name()), + "Storage table identifier must be in the same catalog as the view." + + " Found storage table in catalog: %s, expected: %s", + Array[Object](storageTableCatalogName, catalog.name()) + ) + catalogAndIdentifier.identifier() + } + case None => MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier(ident) + } - // Create the storage table in the Hadoop catalog so it is explicitly registered in the Spark catalog - val tables: HadoopTables = new HadoopTables(new Configuration()) - val icebergSchema = SparkSchemaUtil.convert(viewSchema) // TODO: Add support for partitioning the storage table - val spec: PartitionSpec = PartitionSpec.builderFor(icebergSchema).build - - val table: iceberg.Table = tables.create(icebergSchema, spec, storageTableLocation) + catalog.asInstanceOf[SparkCatalog].createTable( + sparkStorageTableIdentifier, + viewSchema, new Array[Transform](0), ImmutableMap.of[String, String]() + ) + // Capture base table state before inserting into the storage table val baseTables = MaterializedViewUtil.extractBaseTables(queryText).asScala.toList val baseTableSnapshots = getBaseTableSnapshots(baseTables) val baseTableSnapshotsProperties = baseTableSnapshots.map{ @@ -78,19 +87,16 @@ case class CreateMaterializedViewExec( ) -> value.toString } - session.sql(queryText).write.format("iceberg").option( - SparkWriteOptions.WRITE_FORMAT, FileFormat.PARQUET.toString - ).mode(SaveMode.Append).save(storageTableLocation) - - val updateProperties = table.updateProperties() - baseTableSnapshotsProperties.foreach { - case (key, value) => updateProperties.set(key, value) - } - updateProperties.commit() + // Insert into the storage table + session.sql("INSERT INTO " + sparkStorageTableIdentifier + " " + queryText) - table.refresh() + // Update the base table snapshots properties + val baseTablePropertyChanges = baseTableSnapshotsProperties.map{ + case (key, value) => TableChange.setProperty(key, value) + }.toArray - createMaterializedView(storageTableLocation) + catalog.asInstanceOf[SparkCatalog].alterTable(sparkStorageTableIdentifier, baseTablePropertyChanges:_*) + createMaterializedView(sparkStorageTableIdentifier.toString) Nil } @@ -98,7 +104,7 @@ case class CreateMaterializedViewExec( s"CreateMaterializedViewExec: ${ident}" } - private def createMaterializedView(storageTableLocation: String): Unit = { + private def createMaterializedView(storageTableIdentifier: String): Unit = { val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name val currentCatalog = if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null val currentNamespace = session.sessionState.catalogManager.currentNamespace @@ -109,7 +115,7 @@ case class CreateMaterializedViewExec( (ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion, ViewCatalog.PROP_ENGINE_VERSION -> engineVersion) + (MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY -> "true") + - (MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY -> storageTableLocation) + (MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_TABLE_PROPERTY_KEY -> storageTableIdentifier) if (replace) { // CREATE OR REPLACE VIEW diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala index 551e802b6bf8..b57473c36383 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala @@ -18,14 +18,14 @@ */ package org.apache.spark.sql.execution.datasources.v2 -import org.apache.hadoop.conf.Configuration import org.apache.iceberg.catalog.Namespace import org.apache.iceberg.catalog.TableIdentifier import org.apache.iceberg.exceptions -import org.apache.iceberg.hadoop.HadoopTables import org.apache.iceberg.spark.MaterializedViewUtil +import org.apache.iceberg.spark.Spark3Util import org.apache.iceberg.spark.SparkCatalog import org.apache.iceberg.view.View +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.NoSuchViewException import org.apache.spark.sql.catalyst.expressions.Attribute @@ -59,10 +59,12 @@ case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boo )).getOrElse("false").equals("true")) { // get the storage table location then drop the storage table val storageTableLocation = viewProperties.get( - MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY + MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_TABLE_PROPERTY_KEY ) - val tables: HadoopTables = new HadoopTables(new Configuration()) - tables.dropTable(storageTableLocation) + val storageTableIdentifier = Spark3Util.catalogAndIdentifier( + SparkSession.active, storageTableLocation).identifier() + // get active spark session + catalog.asInstanceOf[SparkCatalog].dropTable(storageTableIdentifier) } } case _ => diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala index 2549bd61bf8e..f3a23d006bbb 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala @@ -149,35 +149,48 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi allowExisting, replace, _, - materialized, + Some(materializedViewOptions), _) => - if (materialized) { - CreateMaterializedViewExec( - catalog = viewCatalog, - ident = ident, - queryText = queryText, - columnAliases = columnAliases, - columnComments = columnComments, - queryColumnNames = queryColumnNames, - viewSchema = query.schema, - comment = comment, - properties = properties, - allowExisting = allowExisting, - replace = replace) :: Nil - } else { - CreateV2ViewExec( - catalog = viewCatalog, - ident = ident, - queryText = queryText, - columnAliases = columnAliases, - columnComments = columnComments, - queryColumnNames = queryColumnNames, - viewSchema = query.schema, - comment = comment, - properties = properties, - allowExisting = allowExisting, - replace = replace) :: Nil - } + CreateMaterializedViewExec( + catalog = viewCatalog, + ident = ident, + queryText = queryText, + columnAliases = columnAliases, + columnComments = columnComments, + queryColumnNames = queryColumnNames, + viewSchema = query.schema, + comment = comment, + properties = properties, + allowExisting = allowExisting, + replace = replace, + storageTableIdentifier = materializedViewOptions.storageTableIdentifier) :: Nil + + case CreateIcebergView( + ResolvedIdentifier(viewCatalog: ViewCatalog, ident), + queryText, + query, + columnAliases, + columnComments, + queryColumnNames, + comment, + properties, + allowExisting, + replace, + _, + None, + _) => + CreateV2ViewExec( + catalog = viewCatalog, + ident = ident, + queryText = queryText, + columnAliases = columnAliases, + columnComments = columnComments, + queryColumnNames = queryColumnNames, + viewSchema = query.schema, + comment = comment, + properties = properties, + allowExisting = allowExisting, + replace = replace) :: Nil case DescribeRelation(ResolvedV2View(catalog, ident), _, isExtended, output) => DescribeV2ViewExec(output, catalog.loadView(ident), isExtended) :: Nil diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index 99e0cefd7048..f71fbce9a35c 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -26,12 +26,20 @@ import java.io.IOException; import java.nio.file.Files; import java.util.Map; -import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.TableOperations; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.RuntimeIOException; +import org.apache.iceberg.inmemory.InMemoryCatalog; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.spark.MaterializedViewUtil; -import org.apache.iceberg.spark.Spark3Util; import org.apache.iceberg.spark.SparkCatalogConfig; +import org.apache.iceberg.spark.source.SparkMaterializedView; +import org.apache.iceberg.spark.source.SparkView; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; import org.apache.spark.sql.connector.catalog.CatalogPlugin; @@ -65,15 +73,30 @@ public void removeTable() { @Parameterized.Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") public static Object[][] parameters() { + Map properties = + Maps.newHashMap(SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.properties()); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, "file:" + getTempWarehouseDir()); + properties.put(CatalogProperties.CATALOG_IMPL, InMemoryCatalogWithLocalFileIO.class.getName()); return new Object[][] { { - SparkCatalogConfig.SPARK_WITH_VIEWS.catalogName(), - SparkCatalogConfig.SPARK_WITH_VIEWS.implementation(), - SparkCatalogConfig.SPARK_WITH_VIEWS.properties() + SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.catalogName(), + SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.implementation(), + properties } }; } + private static String getTempWarehouseDir() { + try { + File tempDir = Files.createTempDirectory("warehouse-").toFile(); + tempDir.delete(); + return tempDir.getAbsolutePath(); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public TestMaterializedViews( String catalog, String implementation, Map properties) { super(catalog, implementation, properties); @@ -81,35 +104,24 @@ public TestMaterializedViews( @Test public void assertReadFromStorageTableWhenFresh() throws IOException { - File location = Files.createTempDirectory("materialized-view-test").toFile(); sql("DROP VIEW IF EXISTS %s", materializedViewName); - sql( - "CREATE MATERIALIZED VIEW %s TBLPROPERTIES ('location' = '%s') AS SELECT id, data FROM %s", - materializedViewName, location.getAbsolutePath(), tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); // Assert that number of records in the materialized view is the same as the number of records // in the table assertThat(sql("SELECT * FROM %s", materializedViewName).size()) .isEqualTo(sql("SELECT * FROM %s", tableName).size()); - // Assert that the catalog loadView method returns NoSuchViewException because the view is fresh - assertThatThrownBy( - () -> - sparkViewCatalog() - .loadView(Identifier.of(new String[] {"default"}, materializedViewName))) - .isInstanceOf(NoSuchViewException.class); + // Assert that the catalog loadView method throws IllegalStateException because the view is + // fresh + assertThatThrownBy(() -> sparkViewCatalog().loadView(viewIdentifier())) + .isInstanceOf(IllegalStateException.class); - // Assert that the catalog loadTable method returns the materialized view storage table + // Assert that the catalog loadTable method returns an object, and its type is + // SparkMaterializedView try { - assertThat( - sparkTableCatalog() - .loadTable(Identifier.of(new String[] {"default"}, materializedViewName)) - .name()) - .isEqualTo( - icebergViewCatalog() - .loadView(TableIdentifier.of("default", materializedViewName)) - .properties() - .get(MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY)); + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); } catch (NoSuchTableException e) { fail("Materialized view storage table not found"); } @@ -117,10 +129,7 @@ public void assertReadFromStorageTableWhenFresh() throws IOException { @Test public void assertNotReadFromStorageTableWhenStale() throws IOException { - File location = Files.createTempDirectory("materialized-view-test").toFile(); - sql( - "CREATE MATERIALIZED VIEW %s TBLPROPERTIES ('location' = '%s') AS SELECT id, data FROM %s", - materializedViewName, location.getAbsolutePath(), tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); // Insert one row to the table so the materialized view becomes stale sql("INSERT INTO %s VALUES (1, 'a')", tableName); @@ -130,37 +139,40 @@ public void assertNotReadFromStorageTableWhenStale() throws IOException { assertThat(sql("SELECT * FROM %s", materializedViewName).size()) .isEqualTo(sql("SELECT * FROM %s", tableName).size()); - // Assert that the catalog loadView method returns the view object + // Assert that the catalog loadView method returns an object, and of type SparkView try { - assertThat( - sparkViewCatalog() - .loadView(Identifier.of(new String[] {"default"}, materializedViewName)) - .name()) - .isEqualTo( - icebergViewCatalog() - .loadView(TableIdentifier.of("default", materializedViewName)) - .name()); + assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); } catch (NoSuchViewException e) { fail("Materialized view not found"); } // Assert that the catalog loadTable fails with NoSuchTableException because the view is stale - assertThatThrownBy( - () -> - sparkTableCatalog() - .loadTable(Identifier.of(new String[] {"default"}, materializedViewName))) + assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) .isInstanceOf(NoSuchTableException.class); } @Test - public void assertShowTablesDoesNotShowStorageTable() throws IOException { - File location = Files.createTempDirectory("materialized-view-test").toFile(); + public void testDefaultStorageTableIdentifier() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // Assert that the storage table is in the list of tables + final String materializedViewStorageTableName = + MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier( + Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName)) + .name(); + assertThat(sql("SHOW TABLES")) + .anySatisfy(row -> assertThat(row[1]).isEqualTo(materializedViewStorageTableName)); + } + + @Test + public void testStoredAsClause() { + String customTableName = "custom_table_name"; sql( - "CREATE MATERIALIZED VIEW %s TBLPROPERTIES ('location' = '%s') AS SELECT id, data FROM %s", - materializedViewName, location.getAbsolutePath(), tableName); + "CREATE MATERIALIZED VIEW %s STORED AS '%s' AS SELECT id, data FROM %s", + materializedViewName, customTableName, tableName); - // Assert that the storage table is not shown in the list of tables - assertThat(sql("SHOW TABLES").size() == 2); + // Assert that the storage table with the custom name is in the list of tables + assertThat(sql("SHOW TABLES")).anySatisfy(row -> assertThat(row[1]).isEqualTo(customTableName)); } private ViewCatalog sparkViewCatalog() { @@ -173,10 +185,49 @@ private TableCatalog sparkTableCatalog() { return (TableCatalog) catalogPlugin; } - private org.apache.iceberg.catalog.ViewCatalog icebergViewCatalog() { - Catalog icebergCatalog = Spark3Util.loadIcebergCatalog(spark, catalogName); - assertThat(icebergCatalog).isInstanceOf(org.apache.iceberg.catalog.ViewCatalog.class); - return (org.apache.iceberg.catalog.ViewCatalog) icebergCatalog; + private Identifier viewIdentifier() { + return Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName); + } + + // Required to be public since it is loaded by org.apache.iceberg.CatalogUtil.loadCatalog + public static class InMemoryCatalogWithLocalFileIO extends InMemoryCatalog { + private FileIO localFileIO; + + @Override + public void initialize(String name, Map properties) { + super.initialize(name, properties); + localFileIO = new LocalFileIO(); + } + + @Override + protected TableOperations newTableOps(TableIdentifier tableIdentifier) { + return new InMemoryTableOperations(localFileIO, tableIdentifier); + } + + @Override + protected InMemoryCatalog.InMemoryViewOperations newViewOps(TableIdentifier identifier) { + return new InMemoryViewOperations(localFileIO, identifier); + } + } + + private static class LocalFileIO implements FileIO { + + @Override + public InputFile newInputFile(String path) { + return org.apache.iceberg.Files.localInput(path); + } + + @Override + public OutputFile newOutputFile(String path) { + return org.apache.iceberg.Files.localOutput(path); + } + + @Override + public void deleteFile(String path) { + if (!new File(path).delete()) { + throw new RuntimeIOException("Failed to delete file: " + path); + } + } } // TODO Add DROP MATERIALIZED VIEW test diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java index f6dc101fe4fb..3d4114a1afe0 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java @@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation; import org.apache.spark.sql.catalyst.parser.ParseException; import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; +import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.Table; import org.apache.spark.sql.connector.catalog.TableCatalog; import scala.collection.JavaConverters; @@ -36,9 +37,11 @@ public class MaterializedViewUtil { private MaterializedViewUtil() {} public static final String MATERIALIZED_VIEW_PROPERTY_KEY = "iceberg.materialized.view"; - public static final String MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY = - "iceberg.materialized.view.storage.location"; - public static final String MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX = "base.snapshot."; + public static final String MATERIALIZED_VIEW_STORAGE_TABLE_PROPERTY_KEY = + "iceberg.materialized.view.storage.table"; + public static final String MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX = + "iceberg.base.snapshot."; + private static final String MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX = ".storage.table"; public static List
extractBaseTables(String query) { return extractBaseTableIdentifiers(query).stream() @@ -88,4 +91,11 @@ public static Optional
toSparkTable(List multipartIdent) { } return Optional.empty(); } + + public static Identifier getDefaultMaterializedViewStorageTableIdentifier( + Identifier viewIdentifier) { + return Identifier.of( + viewIdentifier.namespace(), + viewIdentifier.name() + MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX); + } } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index a71323adaef8..c595017a6a80 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; @@ -79,6 +80,7 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException; import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException; +import org.apache.spark.sql.catalyst.parser.ParseException; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.NamespaceChange; import org.apache.spark.sql.connector.catalog.StagedTable; @@ -595,17 +597,11 @@ public View loadView(Identifier ident) throws NoSuchViewException { if (null != asViewCatalog) { try { org.apache.iceberg.view.View view = asViewCatalog.loadView(buildIdentifier(ident)); - // Check if the view is a materialized view. If it is, and storage table is fresh, return - // NoSuchViewException so - // loadTable is attempted instead. - if (view.properties() - .get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY) - .equals("true")) { - if (isFresh(view)) { - throw new NoSuchViewException(ident); - } else { - return new SparkView(catalogName, view); - } + // Check if the view is a materialized view. If it is, and storage table is fresh, throw + // IllegalStateException + if (isMaterializedView(view) && isFresh(view)) { + throw new IllegalStateException( + "Materialized view is fresh. loadTable should be attempted instead."); } else { return new SparkView(catalogName, view); } @@ -617,46 +613,67 @@ public View loadView(Identifier ident) throws NoSuchViewException { throw new NoSuchViewException(ident); } - private boolean isFresh(org.apache.iceberg.view.View view) { + // Candidate to be moved to org.apache.iceberg.view.View + private boolean isMaterializedView(org.apache.iceberg.view.View view) { + return Optional.of(view.properties().get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY)) + .orElse("false") + .equals("true"); + } + + // Candidate to be moved to org.apache.iceberg.view.View + private String getStorageTableIdentifier(org.apache.iceberg.view.View view) { + String identifier = + view.properties().get(MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_TABLE_PROPERTY_KEY); Preconditions.checkState( - view.properties().get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY).equals("true"), - "Cannot check freshness of non-materialized view."); - String storageTableLocation = - view.properties().get(MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY); + identifier != null, "Storage table identifier is not set for materialized view."); + return identifier; + } + + // Candidate to be moved to org.apache.iceberg.view.View but requires loadTable + private Table loadStorageTable(org.apache.iceberg.view.View view) { + String storageTableIdentifier = getStorageTableIdentifier(view); try { - Table storageTable = loadTable(new PathIdentifier(storageTableLocation)); - Map baseTableSnapshotsProperties = - storageTable.properties().entrySet().stream() - .filter( - entry -> - entry - .getKey() - .startsWith( - MaterializedViewUtil - .MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - List
baseTables = MaterializedViewUtil.extractBaseTables(view.sqlFor("spark").sql()); - - for (Table baseTable : baseTables) { - org.apache.iceberg.Table icebergBaseTable = ((SparkTable) baseTable).table(); - String snapshotId = - String.valueOf( - icebergBaseTable.currentSnapshot() == null - ? 0 - : icebergBaseTable.currentSnapshot().snapshotId()); - if (!baseTableSnapshotsProperties - .get( - MaterializedViewUtil.MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX - + icebergBaseTable.uuid()) - .equals(snapshotId)) { - return false; - } + SparkSession session = SparkSession.active(); + Table storageTable = + loadTable(Spark3Util.catalogAndIdentifier(session, storageTableIdentifier).identifier()); + return storageTable; + } catch (ParseException | NoSuchTableException e) { + throw new IllegalStateException("Unable to load storage table for materialized view.", e); + } + } + + // Candidate to be moved to org.apache.iceberg.view.View but requires loadTable + // Second option is to move to SparkMaterializedView + private boolean isFresh(org.apache.iceberg.view.View view) { + Table storageTable = loadStorageTable(view); + Map baseTableSnapshotsProperties = + storageTable.properties().entrySet().stream() + .filter( + entry -> + entry + .getKey() + .startsWith( + MaterializedViewUtil + .MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + List
baseTables = MaterializedViewUtil.extractBaseTables(view.sqlFor("spark").sql()); + + for (Table baseTable : baseTables) { + org.apache.iceberg.Table icebergBaseTable = ((SparkTable) baseTable).table(); + String snapshotId = + String.valueOf( + icebergBaseTable.currentSnapshot() == null + ? 0 + : icebergBaseTable.currentSnapshot().snapshotId()); + if (!baseTableSnapshotsProperties + .get( + MaterializedViewUtil.MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX + + icebergBaseTable.uuid()) + .equals(snapshotId)) { + return false; } - return true; - } catch (NoSuchTableException e) { - throw new IllegalStateException( - "Could not load materialized view storage table from catalog.", e); } + return true; } @Override @@ -691,16 +708,9 @@ public View createView( .withLocation(properties.get("location")) .withProperties(props) .create(); - if (props.get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY).equals("true")) { - String storageTableLocation = - properties.get(MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY); - try { - Table storageTable = loadTable(new PathIdentifier(storageTableLocation)); - return new SparkMaterializedView(catalogName, view, storageTable); - } catch (NoSuchTableException e) { - throw new IllegalStateException( - "Could not load materialized view storage table from catalog.", e); - } + if (isMaterializedView(view)) { + Table storageTable = loadStorageTable(view); + return new SparkMaterializedView(catalogName, view, storageTable); } else { return new SparkView(catalogName, view); } @@ -968,18 +978,9 @@ private Table load(Identifier ident) { if (null != asViewCatalog) { try { org.apache.iceberg.view.View view = asViewCatalog.loadView(buildIdentifier(ident)); - if (view.properties() - .get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY) - .equals("true")) { - if (isFresh(view)) { - String storageTableLocation = - view.properties() - .get(MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_LOCATION_PROPERTY_KEY); - return new SparkMaterializedView( - catalogName, - view, - loadFromPathIdentifier(new PathIdentifier(storageTableLocation))); - } + if (isMaterializedView(view) && isFresh(view)) { + Table storageTable = loadStorageTable(view); + return new SparkMaterializedView(catalogName, view, storageTable); } } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { // Ignore. Just process as a normal table. diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java index 2350aab09b64..38802fb7f76e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java @@ -68,7 +68,11 @@ public enum SparkCatalogConfig { SPARK_WITH_HIVE_VIEWS( "spark_hive_with_views", SparkCatalog.class.getName(), - ImmutableMap.of("type", "hive", "default-namespace", "default", "cache-enabled", "false")); + ImmutableMap.of("type", "hive", "default-namespace", "default", "cache-enabled", "false")), + SPARK_WITH_MATERIALIZED_VIEWS( + "spark_with_materialized_views", + SparkCatalog.class.getName(), + ImmutableMap.of("default-namespace", "default", "cache-enabled", "false")); private final String catalogName; private final String implementation; From e0deb0bdde1516434f343b2898e8305638d093f1 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 12 Mar 2024 22:38:22 -0700 Subject: [PATCH 03/22] Add support for replacing view version --- .../iceberg/view/ViewVersionReplace.java | 9 --- .../v2/CreateMaterializedViewExec.scala | 76 ++++++++++++------- .../iceberg/spark/MaterializedViewUtil.java | 2 + .../apache/iceberg/spark/SparkCatalog.java | 28 +++++-- 4 files changed, 73 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/view/ViewVersionReplace.java b/core/src/main/java/org/apache/iceberg/view/ViewVersionReplace.java index 0150fd2a2a44..8b3d087940a5 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewVersionReplace.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewVersionReplace.java @@ -28,7 +28,6 @@ import static org.apache.iceberg.TableProperties.COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT; import java.util.List; -import java.util.Optional; import org.apache.iceberg.EnvironmentContext; import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.Namespace; @@ -57,14 +56,6 @@ public ViewVersion apply() { } ViewMetadata internalApply() { - // Replacing a materialized view is not supported because the old storage location will wrongly - // transfer to the new version - // if not handled properly. - Preconditions.checkState( - Optional.ofNullable(base.properties().get("iceberg.materialized.view")) - .orElse("false") - .equals("false"), - "Cannot replace a materialized view with a new version"); Preconditions.checkState( !representations.isEmpty(), "Cannot replace view without specifying a query"); Preconditions.checkState(null != schema, "Cannot replace view without specifying schema"); diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala index bcff93dd1ab7..9b72a9d4e99b 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -27,12 +27,14 @@ import org.apache.iceberg.spark.MaterializedViewUtil import org.apache.iceberg.spark.Spark3Util import org.apache.iceberg.spark.SparkCatalog import org.apache.iceberg.spark.source.SparkTable +import org.apache.iceberg.spark.source.SparkView import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.connector.catalog.Table import org.apache.spark.sql.connector.catalog.TableChange +import org.apache.spark.sql.connector.catalog.View import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.types.StructType @@ -72,31 +74,42 @@ case class CreateMaterializedViewExec( case None => MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier(ident) } - // TODO: Add support for partitioning the storage table - catalog.asInstanceOf[SparkCatalog].createTable( - sparkStorageTableIdentifier, - viewSchema, new Array[Transform](0), ImmutableMap.of[String, String]() - ) - - // Capture base table state before inserting into the storage table - val baseTables = MaterializedViewUtil.extractBaseTables(queryText).asScala.toList - val baseTableSnapshots = getBaseTableSnapshots(baseTables) - val baseTableSnapshotsProperties = baseTableSnapshots.map{ - case (key, value) => ( - MaterializedViewUtil.MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX + key.toString - ) -> value.toString - } + val view = createView(sparkStorageTableIdentifier.toString) + + view match { + case Some(v) => { + // TODO: Add support for partitioning the storage table + catalog.asInstanceOf[SparkCatalog].createTable( + sparkStorageTableIdentifier, + viewSchema, new Array[Transform](0), ImmutableMap.of[String, String]() + ) + + // Capture base table state before inserting into the storage table + val baseTables = MaterializedViewUtil.extractBaseTables(queryText).asScala.toList + val baseTableSnapshots = getBaseTableSnapshots(baseTables) + val baseTableSnapshotsProperties = baseTableSnapshots.map { + case (key, value) => ( + MaterializedViewUtil.MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX + key.toString + ) -> value.toString + } + + + val storageTableProperties = baseTableSnapshotsProperties + + (MaterializedViewUtil.MATERIALIZED_VIEW_VERSION_PROPERTY_KEY -> getViewVersion(v).toString) + + // Insert into the storage table + session.sql("INSERT INTO " + sparkStorageTableIdentifier + " " + queryText) - // Insert into the storage table - session.sql("INSERT INTO " + sparkStorageTableIdentifier + " " + queryText) - // Update the base table snapshots properties - val baseTablePropertyChanges = baseTableSnapshotsProperties.map{ - case (key, value) => TableChange.setProperty(key, value) - }.toArray + // Update the storage table properties + val storageTablePropertyChanges = storageTableProperties.map { + case (key, value) => TableChange.setProperty(key, value) + }.toArray - catalog.asInstanceOf[SparkCatalog].alterTable(sparkStorageTableIdentifier, baseTablePropertyChanges:_*) - createMaterializedView(sparkStorageTableIdentifier.toString) + catalog.asInstanceOf[SparkCatalog].alterTable(sparkStorageTableIdentifier, storageTablePropertyChanges: _*) + } + case None => + } Nil } @@ -104,7 +117,7 @@ case class CreateMaterializedViewExec( s"CreateMaterializedViewExec: ${ident}" } - private def createMaterializedView(storageTableIdentifier: String): Unit = { + private def createView(storageTableIdentifier: String): Option[View] = { val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name val currentCatalog = if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null val currentNamespace = session.sessionState.catalogManager.currentNamespace @@ -123,7 +136,7 @@ case class CreateMaterializedViewExec( catalog.dropView(ident) } // FIXME: replaceView API doesn't exist in Spark 3.5 - catalog.createView( + val view = catalog.createView( ident, queryText, currentCatalog, @@ -133,10 +146,11 @@ case class CreateMaterializedViewExec( columnAliases.toArray, columnComments.map(c => c.orNull).toArray, newProperties.asJava) + Some(view) } else { try { // CREATE VIEW [IF NOT EXISTS] - catalog.createView( + val view = catalog.createView( ident, queryText, currentCatalog, @@ -146,9 +160,10 @@ case class CreateMaterializedViewExec( columnAliases.toArray, columnComments.map(c => c.orNull).toArray, newProperties.asJava) + Some(view) } catch { // TODO: Make sure the existing view is also a materialized view - case _: ViewAlreadyExistsException if allowExisting => // Ignore + case _: ViewAlreadyExistsException if allowExisting => None } } } @@ -163,4 +178,13 @@ case class CreateMaterializedViewExec( throw new UnsupportedOperationException("Only Spark tables are supported") }.toMap } + + private def getViewVersion(view: View): Long = { + view match { + case sparkView: SparkView => + sparkView.view().currentVersion().versionId() + case _ => + throw new UnsupportedOperationException("Only Spark views are supported") + } + } } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java index 3d4114a1afe0..633b705eb37a 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java @@ -41,6 +41,8 @@ private MaterializedViewUtil() {} "iceberg.materialized.view.storage.table"; public static final String MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX = "iceberg.base.snapshot."; + public static final String MATERIALIZED_VIEW_VERSION_PROPERTY_KEY = + "iceberg.materialized.view.version"; private static final String MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX = ".storage.table"; public static List
extractBaseTables(String query) { diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index c595017a6a80..3440a75081e9 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -646,8 +646,27 @@ private Table loadStorageTable(org.apache.iceberg.view.View view) { // Second option is to move to SparkMaterializedView private boolean isFresh(org.apache.iceberg.view.View view) { Table storageTable = loadStorageTable(view); + Map storageTableProperties = storageTable.properties(); + + // Get the parent view version id from the storage table properties + String storageTableViewVersionIdPropertyValue = + storageTableProperties.get( + MaterializedViewUtil.MATERIALIZED_VIEW_VERSION_PROPERTY_KEY); + if (storageTableViewVersionIdPropertyValue == null) { + throw new IllegalStateException( + "Storage table properties do not contain the virtual view version id property."); + } + int storageTableViewVersionId = Integer.parseInt(storageTableViewVersionIdPropertyValue); + + // If the storage table view version id is different from the current version id, the + // materialized view is not fresh + if (storageTableViewVersionId != view.currentVersion().versionId()) { + return false; + } + + // Get the base table snapshot ids from the storage table properties Map baseTableSnapshotsProperties = - storageTable.properties().entrySet().stream() + storageTableProperties.entrySet().stream() .filter( entry -> entry @@ -708,12 +727,7 @@ public View createView( .withLocation(properties.get("location")) .withProperties(props) .create(); - if (isMaterializedView(view)) { - Table storageTable = loadStorageTable(view); - return new SparkMaterializedView(catalogName, view, storageTable); - } else { - return new SparkView(catalogName, view); - } + return new SparkView(catalogName, view); } catch (org.apache.iceberg.exceptions.NoSuchNamespaceException e) { throw new NoSuchNamespaceException(currentNamespace); } catch (AlreadyExistsException e) { From e32a021d4fd1de0942f7f989975ccba993e80734 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 9 Jul 2024 16:22:06 -0700 Subject: [PATCH 04/22] Update MV implementation to use new spec elements --- .../java/org/apache/iceberg/view/View.java | 5 ++ .../org/apache/iceberg/view/ViewBuilder.java | 9 +++ .../view/BaseMetastoreViewCatalog.java | 7 ++ .../org/apache/iceberg/view/BaseView.java | 6 ++ .../v2/CreateMaterializedViewExec.scala | 77 ++++++++++--------- .../apache/iceberg/spark/SparkCatalog.java | 3 +- 6 files changed, 69 insertions(+), 38 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/view/View.java b/api/src/main/java/org/apache/iceberg/view/View.java index 779592d03104..e6b82d6b3153 100644 --- a/api/src/main/java/org/apache/iceberg/view/View.java +++ b/api/src/main/java/org/apache/iceberg/view/View.java @@ -23,6 +23,7 @@ import java.util.UUID; import org.apache.iceberg.Schema; import org.apache.iceberg.UpdateLocation; +import org.apache.iceberg.catalog.TableIdentifier; /** Interface for view definition. */ public interface View { @@ -88,6 +89,10 @@ default String location() { throw new UnsupportedOperationException("Retrieving a view's location is not supported"); } + default TableIdentifier storageTableIdentifier() { + throw new UnsupportedOperationException("Retrieving a view's storage table identifier is not supported"); + } + /** * Create a new {@link UpdateViewProperties} to update view properties. * diff --git a/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java b/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java index 0717e492fc58..4809878ba958 100644 --- a/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java +++ b/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java @@ -19,6 +19,8 @@ package org.apache.iceberg.view; import java.util.Map; + +import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.catalog.ViewCatalog; /** @@ -55,6 +57,13 @@ default ViewBuilder withLocation(String location) { throw new UnsupportedOperationException("Setting a view's location is not supported"); } + /** Set the storage table identifier in case of a materialized view. + * + * @param storageTableIdentifier the storage table identifier + * @return this for method chaining + */ + ViewBuilder withStorageTableIdentifier(TableIdentifier storageTableIdentifier); + /** * Create the view. * diff --git a/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java b/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java index cb28577b4983..b90e897900c9 100644 --- a/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java +++ b/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java @@ -80,6 +80,7 @@ protected class BaseViewBuilder implements ViewBuilder { private String defaultCatalog = null; private Schema schema = null; private String location = null; + private TableIdentifier storageTableIdentifier = null; protected BaseViewBuilder(TableIdentifier identifier) { Preconditions.checkArgument( @@ -159,6 +160,12 @@ public ViewBuilder withLocation(String newLocation) { return this; } + @Override + public ViewBuilder withStorageTableIdentifier(TableIdentifier storageTableIdentifier) { + this.storageTableIdentifier = storageTableIdentifier; + return this; + } + @Override public View create() { return create(newViewOps(identifier)); diff --git a/core/src/main/java/org/apache/iceberg/view/BaseView.java b/core/src/main/java/org/apache/iceberg/view/BaseView.java index d30fc6535098..b579548f7c51 100644 --- a/core/src/main/java/org/apache/iceberg/view/BaseView.java +++ b/core/src/main/java/org/apache/iceberg/view/BaseView.java @@ -24,6 +24,7 @@ import java.util.UUID; import org.apache.iceberg.Schema; import org.apache.iceberg.UpdateLocation; +import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; public class BaseView implements View, Serializable { @@ -85,6 +86,11 @@ public String location() { return operations().current().location(); } + @Override + public TableIdentifier storageTableIdentifier() { + return operations().current().storageTableIdentifier(); + } + @Override public UpdateViewProperties updateProperties() { return new PropertiesUpdate(ops); diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala index 9b72a9d4e99b..6ccc63c96f5b 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -20,14 +20,14 @@ package org.apache.spark.sql.execution.datasources.v2 +import org.apache.iceberg.{SnapshotUpdate, catalog} + import java.util.UUID +import org.apache.iceberg.catalog.{Namespace, TableIdentifier} import org.apache.iceberg.relocated.com.google.common.base.Preconditions import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap -import org.apache.iceberg.spark.MaterializedViewUtil -import org.apache.iceberg.spark.Spark3Util -import org.apache.iceberg.spark.SparkCatalog -import org.apache.iceberg.spark.source.SparkTable -import org.apache.iceberg.spark.source.SparkView +import org.apache.iceberg.spark.{MaterializedViewUtil, Spark3Util, SparkCatalog, SparkSchemaUtil} +import org.apache.iceberg.spark.source.{SparkTable, SparkView} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException import org.apache.spark.sql.catalyst.expressions.Attribute @@ -38,6 +38,7 @@ import org.apache.spark.sql.connector.catalog.View import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.types.StructType + import scala.collection.JavaConverters._ case class CreateMaterializedViewExec( @@ -58,7 +59,7 @@ case class CreateMaterializedViewExec( override protected def run(): Seq[InternalRow] = { - // Check if storageTableIdentifier is provided, if not, generate a default identifier + // Check if storageTableIdentifier is provided. If not, generate a default identifier. val sparkStorageTableIdentifier = storageTableIdentifier match { case Some(identifier) => { val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(session, identifier) @@ -66,7 +67,7 @@ case class CreateMaterializedViewExec( Preconditions.checkState( storageTableCatalogName.equals(catalog.name()), "Storage table identifier must be in the same catalog as the view." + - " Found storage table in catalog: %s, expected: %s", + " Found storage table in catalog: %s, expected: %s.", Array[Object](storageTableCatalogName, catalog.name()) ) catalogAndIdentifier.identifier() @@ -100,13 +101,11 @@ case class CreateMaterializedViewExec( // Insert into the storage table session.sql("INSERT INTO " + sparkStorageTableIdentifier + " " + queryText) - - // Update the storage table properties - val storageTablePropertyChanges = storageTableProperties.map { - case (key, value) => TableChange.setProperty(key, value) - }.toArray - - catalog.asInstanceOf[SparkCatalog].alterTable(sparkStorageTableIdentifier, storageTablePropertyChanges: _*) + // Load the storage table as an Iceberg table + val icebergStorageTable = catalog.asInstanceOf[org.apache.iceberg.catalog.Catalog].loadTable(TableIdentifier.parse(sparkStorageTableIdentifier.toString)) + val replaceSnapshot = icebergStorageTable.newRewrite() + replaceSnapshot.set("mv-base-table-snapshots", baseTableSnapshots.asJava.toString) + replaceSnapshot.commit() } case None => } @@ -118,6 +117,7 @@ case class CreateMaterializedViewExec( } private def createView(storageTableIdentifier: String): Option[View] = { + val icebergSchema = SparkSchemaUtil.convert(viewSchema) val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name val currentCatalog = if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null val currentNamespace = session.sessionState.catalogManager.currentNamespace @@ -128,7 +128,9 @@ case class CreateMaterializedViewExec( (ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion, ViewCatalog.PROP_ENGINE_VERSION -> engineVersion) + (MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY -> "true") + - (MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_TABLE_PROPERTY_KEY -> storageTableIdentifier) + (MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_TABLE_PROPERTY_KEY -> storageTableIdentifier) + + ("queryColumnNames" -> queryColumnNames.mkString(",")) + if (replace) { // CREATE OR REPLACE VIEW @@ -136,31 +138,32 @@ case class CreateMaterializedViewExec( catalog.dropView(ident) } // FIXME: replaceView API doesn't exist in Spark 3.5 - val view = catalog.createView( - ident, - queryText, - currentCatalog, - currentNamespace, - viewSchema, - queryColumnNames.toArray, - columnAliases.toArray, - columnComments.map(c => c.orNull).toArray, - newProperties.asJava) - Some(view) + val icebergView = catalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( + Spark3Util.identifierToTableIdentifier(ident)) + .withDefaultCatalog(currentCatalog) + .withDefaultNamespace(Namespace.of(currentNamespace: _*)) + .withQuery("spark", queryText) + .withSchema(icebergSchema) + .withLocation(properties.get("location").orNull) + .withProperties(newProperties.asJava) + .withStorageTableIdentifier(TableIdentifier.parse(storageTableIdentifier)) + .create() + Some(new SparkView(catalog.name(), icebergView)) + } else { try { // CREATE VIEW [IF NOT EXISTS] - val view = catalog.createView( - ident, - queryText, - currentCatalog, - currentNamespace, - viewSchema, - queryColumnNames.toArray, - columnAliases.toArray, - columnComments.map(c => c.orNull).toArray, - newProperties.asJava) - Some(view) + val icebergView = catalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( + Spark3Util.identifierToTableIdentifier(ident)) + .withDefaultCatalog(currentCatalog) + .withDefaultNamespace(Namespace.of(currentNamespace: _*)) + .withQuery("spark", queryText) + .withSchema(icebergSchema) + .withLocation(properties.get("location").orNull) + .withProperties(newProperties.asJava) + .withStorageTableIdentifier(TableIdentifier.parse(storageTableIdentifier)) + .create() + Some(new SparkView(catalog.name(), icebergView)) } catch { // TODO: Make sure the existing view is also a materialized view case _: ViewAlreadyExistsException if allowExisting => None diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index 3440a75081e9..0d68e138bd04 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -630,7 +630,8 @@ private String getStorageTableIdentifier(org.apache.iceberg.view.View view) { } // Candidate to be moved to org.apache.iceberg.view.View but requires loadTable - private Table loadStorageTable(org.apache.iceberg.view.View view) { + private org.apache.iceberg.Table loadStorageTable(org.apache.iceberg.view.View view) { + String storageTableIdentifier = vie String storageTableIdentifier = getStorageTableIdentifier(view); try { SparkSession session = SparkSession.active(); From 1fb1e9281379f741f86b6947bd060789b9860d3f Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Wed, 18 Mar 2026 12:36:47 -0700 Subject: [PATCH 05/22] Add refresh-state model; update Spark MV layer --- .../java/org/apache/iceberg/view/View.java | 5 - .../org/apache/iceberg/view/ViewVersion.java | 12 + .../iceberg/rest/RESTSessionCatalog.java | 29 ++- .../view/BaseMetastoreViewCatalog.java | 22 +- .../org/apache/iceberg/view/BaseView.java | 6 - .../apache/iceberg/view/BaseViewVersion.java | 5 + .../org/apache/iceberg/view/RefreshState.java | 55 +++++ .../iceberg/view/RefreshStateParser.java | 146 ++++++++++++ .../org/apache/iceberg/view/SourceState.java | 42 ++++ .../apache/iceberg/view/SourceTableState.java | 89 ++++++++ .../apache/iceberg/view/SourceViewState.java | 81 +++++++ .../iceberg/view/ViewVersionParser.java | 40 +++- .../iceberg/view/TestRefreshStateParser.java | 207 ++++++++++++++++++ .../iceberg/view/TestViewVersionParser.java | 76 +++++++ .../IcebergSparkSqlExtensionsParser.scala | 4 +- .../v2/CreateMaterializedViewExec.scala | 88 +++----- .../datasources/v2/DropV2ViewExec.scala | 24 +- .../extensions/TestMaterializedViews.java | 167 +++++++++++--- .../iceberg/spark/MaterializedViewUtil.java | 70 +----- .../apache/iceberg/spark/SparkCatalog.java | 117 +++++----- 20 files changed, 1003 insertions(+), 282 deletions(-) create mode 100644 core/src/main/java/org/apache/iceberg/view/RefreshState.java create mode 100644 core/src/main/java/org/apache/iceberg/view/RefreshStateParser.java create mode 100644 core/src/main/java/org/apache/iceberg/view/SourceState.java create mode 100644 core/src/main/java/org/apache/iceberg/view/SourceTableState.java create mode 100644 core/src/main/java/org/apache/iceberg/view/SourceViewState.java create mode 100644 core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java diff --git a/api/src/main/java/org/apache/iceberg/view/View.java b/api/src/main/java/org/apache/iceberg/view/View.java index e6b82d6b3153..779592d03104 100644 --- a/api/src/main/java/org/apache/iceberg/view/View.java +++ b/api/src/main/java/org/apache/iceberg/view/View.java @@ -23,7 +23,6 @@ import java.util.UUID; import org.apache.iceberg.Schema; import org.apache.iceberg.UpdateLocation; -import org.apache.iceberg.catalog.TableIdentifier; /** Interface for view definition. */ public interface View { @@ -89,10 +88,6 @@ default String location() { throw new UnsupportedOperationException("Retrieving a view's location is not supported"); } - default TableIdentifier storageTableIdentifier() { - throw new UnsupportedOperationException("Retrieving a view's storage table identifier is not supported"); - } - /** * Create a new {@link UpdateViewProperties} to update view properties. * diff --git a/api/src/main/java/org/apache/iceberg/view/ViewVersion.java b/api/src/main/java/org/apache/iceberg/view/ViewVersion.java index c63aa9ff2e3e..4733f1bdcdf3 100644 --- a/api/src/main/java/org/apache/iceberg/view/ViewVersion.java +++ b/api/src/main/java/org/apache/iceberg/view/ViewVersion.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; /** * A version of the view at a point in time. @@ -78,4 +79,15 @@ default String defaultCatalog() { /** The default namespace to use when the SQL does not contain a namespace. */ Namespace defaultNamespace(); + + /** + * The storage table identifier for materialized views. + * + *

When null, the entity is a regular view. When set, the entity is a materialized view and + * this identifies the storage table that holds the precomputed data. The storage table must be in + * the same catalog as the materialized view. + */ + default TableIdentifier storageTable() { + return null; + } } diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index e7b68dad4aae..4dbe44101720 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -1718,6 +1718,7 @@ private class RESTViewBuilder implements ViewBuilder { private String defaultCatalog = null; private Schema schema = null; private String location = null; + private TableIdentifier storageTableIdentifier = null; private RESTViewBuilder(SessionContext context, TableIdentifier identifier) { checkViewIdentifierIsValid(identifier); @@ -1797,6 +1798,12 @@ public ViewBuilder withLocation(String newLocation) { return this; } + @Override + public ViewBuilder withStorageTableIdentifier(TableIdentifier newStorageTableIdentifier) { + this.storageTableIdentifier = newStorageTableIdentifier; + return this; + } + @Override public View create() { Endpoint.check(endpoints, Endpoint.V1_CREATE_VIEW); @@ -1806,7 +1813,7 @@ public View create() { Preconditions.checkState( null != defaultNamespace, "Cannot create view without specifying a default namespace"); - ViewVersion viewVersion = + ImmutableViewVersion.Builder versionBuilder = ImmutableViewVersion.builder() .versionId(1) .schemaId(schema.schemaId()) @@ -1814,8 +1821,13 @@ public View create() { .defaultNamespace(defaultNamespace) .defaultCatalog(defaultCatalog) .timestampMillis(System.currentTimeMillis()) - .putAllSummary(EnvironmentContext.get()) - .build(); + .putAllSummary(EnvironmentContext.get()); + + if (storageTableIdentifier != null) { + versionBuilder.storageTable(storageTableIdentifier); + } + + ViewVersion viewVersion = versionBuilder.build(); properties.putAll(viewOverrideProperties()); @@ -1906,7 +1918,7 @@ private View replace(LoadViewResponse response) { .max(Integer::compareTo) .orElseGet(metadata::currentVersionId); - ViewVersion viewVersion = + ImmutableViewVersion.Builder versionBuilder = ImmutableViewVersion.builder() .versionId(maxVersionId + 1) .schemaId(schema.schemaId()) @@ -1914,8 +1926,13 @@ private View replace(LoadViewResponse response) { .defaultNamespace(defaultNamespace) .defaultCatalog(defaultCatalog) .timestampMillis(System.currentTimeMillis()) - .putAllSummary(EnvironmentContext.get()) - .build(); + .putAllSummary(EnvironmentContext.get()); + + if (storageTableIdentifier != null) { + versionBuilder.storageTable(storageTableIdentifier); + } + + ViewVersion viewVersion = versionBuilder.build(); properties.putAll(viewOverrideProperties()); diff --git a/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java b/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java index b90e897900c9..174764782ba9 100644 --- a/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java +++ b/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java @@ -197,7 +197,7 @@ private View create(ViewOperations ops) { Preconditions.checkState( null != defaultNamespace, "Cannot create view without specifying a default namespace"); - ViewVersion viewVersion = + ImmutableViewVersion.Builder versionBuilder = ImmutableViewVersion.builder() .versionId(1) .schemaId(schema.schemaId()) @@ -205,8 +205,13 @@ private View create(ViewOperations ops) { .defaultNamespace(defaultNamespace) .defaultCatalog(defaultCatalog) .timestampMillis(System.currentTimeMillis()) - .putAllSummary(EnvironmentContext.get()) - .build(); + .putAllSummary(EnvironmentContext.get()); + + if (storageTableIdentifier != null) { + versionBuilder.storageTable(storageTableIdentifier); + } + + ViewVersion viewVersion = versionBuilder.build(); properties.putAll(viewOverrideProperties()); @@ -248,7 +253,7 @@ private View replace(ViewOperations ops) { .max(Integer::compareTo) .orElseGet(metadata::currentVersionId); - ViewVersion viewVersion = + ImmutableViewVersion.Builder versionBuilder = ImmutableViewVersion.builder() .versionId(maxVersionId + 1) .schemaId(schema.schemaId()) @@ -256,8 +261,13 @@ private View replace(ViewOperations ops) { .defaultNamespace(defaultNamespace) .defaultCatalog(defaultCatalog) .timestampMillis(System.currentTimeMillis()) - .putAllSummary(EnvironmentContext.get()) - .build(); + .putAllSummary(EnvironmentContext.get()); + + if (storageTableIdentifier != null) { + versionBuilder.storageTable(storageTableIdentifier); + } + + ViewVersion viewVersion = versionBuilder.build(); properties.putAll(viewOverrideProperties()); diff --git a/core/src/main/java/org/apache/iceberg/view/BaseView.java b/core/src/main/java/org/apache/iceberg/view/BaseView.java index b579548f7c51..d30fc6535098 100644 --- a/core/src/main/java/org/apache/iceberg/view/BaseView.java +++ b/core/src/main/java/org/apache/iceberg/view/BaseView.java @@ -24,7 +24,6 @@ import java.util.UUID; import org.apache.iceberg.Schema; import org.apache.iceberg.UpdateLocation; -import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; public class BaseView implements View, Serializable { @@ -86,11 +85,6 @@ public String location() { return operations().current().location(); } - @Override - public TableIdentifier storageTableIdentifier() { - return operations().current().storageTableIdentifier(); - } - @Override public UpdateViewProperties updateProperties() { return new PropertiesUpdate(ops); diff --git a/core/src/main/java/org/apache/iceberg/view/BaseViewVersion.java b/core/src/main/java/org/apache/iceberg/view/BaseViewVersion.java index 5c687d9f0085..151ea7b68bd9 100644 --- a/core/src/main/java/org/apache/iceberg/view/BaseViewVersion.java +++ b/core/src/main/java/org/apache/iceberg/view/BaseViewVersion.java @@ -19,6 +19,7 @@ package org.apache.iceberg.view; import javax.annotation.Nullable; +import org.apache.iceberg.catalog.TableIdentifier; import org.immutables.value.Value; /** @@ -45,4 +46,8 @@ default String operation() { @Override @Nullable String defaultCatalog(); + + @Override + @Nullable + TableIdentifier storageTable(); } diff --git a/core/src/main/java/org/apache/iceberg/view/RefreshState.java b/core/src/main/java/org/apache/iceberg/view/RefreshState.java new file mode 100644 index 000000000000..160f280ac79c --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/view/RefreshState.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.view; + +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +/** + * Captures the state of source tables and views at the time of a materialized view refresh + * operation. Stored as a JSON-encoded string in the storage table's snapshot summary under the + * {@code refresh-state} key. + */ +public class RefreshState { + public static final String REFRESH_STATE_SUMMARY_KEY = "refresh-state"; + + private final int viewVersionId; + private final List sourceStates; + private final long refreshStartTimestampMs; + + public RefreshState( + int viewVersionId, List sourceStates, long refreshStartTimestampMs) { + Preconditions.checkArgument(sourceStates != null, "Source states list is required"); + this.viewVersionId = viewVersionId; + this.sourceStates = sourceStates; + this.refreshStartTimestampMs = refreshStartTimestampMs; + } + + public int viewVersionId() { + return viewVersionId; + } + + public List sourceStates() { + return sourceStates; + } + + public long refreshStartTimestampMs() { + return refreshStartTimestampMs; + } +} diff --git a/core/src/main/java/org/apache/iceberg/view/RefreshStateParser.java b/core/src/main/java/org/apache/iceberg/view/RefreshStateParser.java new file mode 100644 index 000000000000..f032f93604db --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/view/RefreshStateParser.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.view; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.util.JsonUtil; + +public class RefreshStateParser { + + private static final String VIEW_VERSION_ID = "view-version-id"; + private static final String SOURCE_STATES = "source-states"; + private static final String REFRESH_START_TIMESTAMP_MS = "refresh-start-timestamp-ms"; + + // Source state common fields + private static final String TYPE = "type"; + private static final String NAME = "name"; + private static final String NAMESPACE = "namespace"; + private static final String CATALOG = "catalog"; + private static final String UUID = "uuid"; + + // Source table state fields + private static final String SNAPSHOT_ID = "snapshot-id"; + private static final String REF = "ref"; + + // Source view state fields + private static final String VERSION_ID = "version-id"; + + private RefreshStateParser() {} + + public static String toJson(RefreshState refreshState) { + return JsonUtil.generate(gen -> toJson(refreshState, gen), false); + } + + public static void toJson(RefreshState refreshState, JsonGenerator generator) throws IOException { + Preconditions.checkArgument(refreshState != null, "Cannot serialize null refresh state"); + generator.writeStartObject(); + + generator.writeNumberField(VIEW_VERSION_ID, refreshState.viewVersionId()); + generator.writeNumberField( + REFRESH_START_TIMESTAMP_MS, refreshState.refreshStartTimestampMs()); + + generator.writeArrayFieldStart(SOURCE_STATES); + for (SourceState sourceState : refreshState.sourceStates()) { + writeSourceState(sourceState, generator); + } + generator.writeEndArray(); + + generator.writeEndObject(); + } + + private static void writeSourceState(SourceState sourceState, JsonGenerator generator) + throws IOException { + generator.writeStartObject(); + generator.writeStringField(TYPE, sourceState.type()); + JsonUtil.writeStringArray(NAMESPACE, sourceState.namespace(), generator); + generator.writeStringField(NAME, sourceState.name()); + + if (sourceState.catalog() != null) { + generator.writeStringField(CATALOG, sourceState.catalog()); + } + + generator.writeStringField(UUID, sourceState.uuid()); + + if (sourceState instanceof SourceTableState) { + SourceTableState tableState = (SourceTableState) sourceState; + generator.writeNumberField(SNAPSHOT_ID, tableState.snapshotId()); + if (tableState.ref() != null) { + generator.writeStringField(REF, tableState.ref()); + } + } else if (sourceState instanceof SourceViewState) { + SourceViewState viewState = (SourceViewState) sourceState; + generator.writeNumberField(VERSION_ID, viewState.versionId()); + } + + generator.writeEndObject(); + } + + public static RefreshState fromJson(String json) { + Preconditions.checkArgument(json != null, "Cannot parse refresh state from null string"); + return JsonUtil.parse(json, RefreshStateParser::fromJson); + } + + public static RefreshState fromJson(JsonNode node) { + Preconditions.checkArgument(node != null, "Cannot parse refresh state from null object"); + Preconditions.checkArgument( + node.isObject(), "Cannot parse refresh state from a non-object: %s", node); + + int viewVersionId = JsonUtil.getInt(VIEW_VERSION_ID, node); + long refreshStartTimestampMs = JsonUtil.getLong(REFRESH_START_TIMESTAMP_MS, node); + + JsonNode sourceStatesNode = node.get(SOURCE_STATES); + ImmutableList.Builder sourceStates = ImmutableList.builder(); + if (sourceStatesNode != null && sourceStatesNode.isArray()) { + for (JsonNode sourceStateNode : sourceStatesNode) { + sourceStates.add(parseSourceState(sourceStateNode)); + } + } + + return new RefreshState(viewVersionId, sourceStates.build(), refreshStartTimestampMs); + } + + private static SourceState parseSourceState(JsonNode node) { + String type = JsonUtil.getString(TYPE, node); + String name = JsonUtil.getString(NAME, node); + List namespace = + Arrays.asList(JsonUtil.getStringArray(JsonUtil.get(NAMESPACE, node))); + String catalog = JsonUtil.getStringOrNull(CATALOG, node); + String uuid = JsonUtil.getString(UUID, node); + + switch (type) { + case SourceTableState.TYPE: + long snapshotId = JsonUtil.getLong(SNAPSHOT_ID, node); + String ref = JsonUtil.getStringOrNull(REF, node); + return new SourceTableState(name, namespace, catalog, uuid, snapshotId, ref); + + case SourceViewState.TYPE: + int versionId = JsonUtil.getInt(VERSION_ID, node); + return new SourceViewState(name, namespace, catalog, uuid, versionId); + + default: + throw new IllegalArgumentException("Unknown source state type: " + type); + } + } +} diff --git a/core/src/main/java/org/apache/iceberg/view/SourceState.java b/core/src/main/java/org/apache/iceberg/view/SourceState.java new file mode 100644 index 000000000000..54449017dec7 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/view/SourceState.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.view; + +import java.util.List; +import javax.annotation.Nullable; + +/** Base type for source state records in a materialized view's refresh state. */ +public interface SourceState { + + /** The type discriminator for this source state record. */ + String type(); + + /** The name of the source object. */ + String name(); + + /** The namespace levels of the source object. */ + List namespace(); + + /** The catalog of the source object, or null if the same as the materialized view's catalog. */ + @Nullable + String catalog(); + + /** The UUID of the source object. */ + String uuid(); +} diff --git a/core/src/main/java/org/apache/iceberg/view/SourceTableState.java b/core/src/main/java/org/apache/iceberg/view/SourceTableState.java new file mode 100644 index 000000000000..adaa3e0ccf11 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/view/SourceTableState.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.view; + +import java.util.List; +import javax.annotation.Nullable; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +/** Captures the state of a source table at the time of a materialized view refresh. */ +public class SourceTableState implements SourceState { + public static final String TYPE = "table"; + + private final String name; + private final List namespace; + private final String catalog; + private final String uuid; + private final long snapshotId; + private final String ref; + + public SourceTableState( + String name, + List namespace, + @Nullable String catalog, + String uuid, + long snapshotId, + @Nullable String ref) { + Preconditions.checkArgument(name != null, "Source table name is required"); + Preconditions.checkArgument( + namespace != null && !namespace.isEmpty(), "Source table namespace is required"); + Preconditions.checkArgument(uuid != null, "Source table uuid is required"); + this.name = name; + this.namespace = namespace; + this.catalog = catalog; + this.uuid = uuid; + this.snapshotId = snapshotId; + this.ref = ref; + } + + @Override + public String type() { + return TYPE; + } + + @Override + public String name() { + return name; + } + + @Override + public List namespace() { + return namespace; + } + + @Override + @Nullable + public String catalog() { + return catalog; + } + + @Override + public String uuid() { + return uuid; + } + + public long snapshotId() { + return snapshotId; + } + + @Nullable + public String ref() { + return ref; + } +} diff --git a/core/src/main/java/org/apache/iceberg/view/SourceViewState.java b/core/src/main/java/org/apache/iceberg/view/SourceViewState.java new file mode 100644 index 000000000000..266dd171f0c9 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/view/SourceViewState.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.view; + +import java.util.List; +import javax.annotation.Nullable; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +/** Captures the state of a source view at the time of a materialized view refresh. */ +public class SourceViewState implements SourceState { + public static final String TYPE = "view"; + + private final String name; + private final List namespace; + private final String catalog; + private final String uuid; + private final int versionId; + + public SourceViewState( + String name, + List namespace, + @Nullable String catalog, + String uuid, + int versionId) { + Preconditions.checkArgument(name != null, "Source view name is required"); + Preconditions.checkArgument( + namespace != null && !namespace.isEmpty(), "Source view namespace is required"); + Preconditions.checkArgument(uuid != null, "Source view uuid is required"); + this.name = name; + this.namespace = namespace; + this.catalog = catalog; + this.uuid = uuid; + this.versionId = versionId; + } + + @Override + public String type() { + return TYPE; + } + + @Override + public String name() { + return name; + } + + @Override + public List namespace() { + return namespace; + } + + @Override + @Nullable + public String catalog() { + return catalog; + } + + @Override + public String uuid() { + return uuid; + } + + public int versionId() { + return versionId; + } +} diff --git a/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java b/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java index 69208ce34062..e5895f206115 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewVersionParser.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.Map; import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.util.JsonUtil; @@ -37,6 +38,9 @@ public class ViewVersionParser { private static final String SCHEMA_ID = "schema-id"; private static final String DEFAULT_CATALOG = "default-catalog"; private static final String DEFAULT_NAMESPACE = "default-namespace"; + private static final String STORAGE_TABLE = "storage-table"; + private static final String NAMESPACE = "namespace"; + private static final String NAME = "name"; private ViewVersionParser() {} @@ -62,6 +66,14 @@ public static void toJson(ViewVersion version, JsonGenerator generator) throws I } generator.writeEndArray(); + if (version.storageTable() != null) { + generator.writeObjectFieldStart(STORAGE_TABLE); + JsonUtil.writeStringArray( + NAMESPACE, Arrays.asList(version.storageTable().namespace().levels()), generator); + generator.writeStringField(NAME, version.storageTable().name()); + generator.writeEndObject(); + } + generator.writeEndObject(); } @@ -98,14 +110,24 @@ public static ViewVersion fromJson(JsonNode node) { Namespace defaultNamespace = Namespace.of(JsonUtil.getStringArray(DEFAULT_NAMESPACE, node)); - return ImmutableViewVersion.builder() - .versionId(versionId) - .timestampMillis(timestamp) - .schemaId(schemaId) - .summary(summary) - .defaultNamespace(defaultNamespace) - .defaultCatalog(defaultCatalog) - .representations(representations.build()) - .build(); + ImmutableViewVersion.Builder builder = + ImmutableViewVersion.builder() + .versionId(versionId) + .timestampMillis(timestamp) + .schemaId(schemaId) + .summary(summary) + .defaultNamespace(defaultNamespace) + .defaultCatalog(defaultCatalog) + .representations(representations.build()); + + if (node.has(STORAGE_TABLE)) { + JsonNode storageTableNode = node.get(STORAGE_TABLE); + Namespace storageNamespace = + Namespace.of(JsonUtil.getStringArray(JsonUtil.get(NAMESPACE, storageTableNode))); + String storageTableName = JsonUtil.getString(NAME, storageTableNode); + builder.storageTable(TableIdentifier.of(storageNamespace, storageTableName)); + } + + return builder.build(); } } diff --git a/core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java b/core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java new file mode 100644 index 000000000000..cc0ab06ca1cd --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.view; + +import java.util.Arrays; +import java.util.Collections; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestRefreshStateParser { + + @Test + public void testRoundTripSourceTableState() { + SourceTableState tableState = + new SourceTableState( + "events", + Arrays.asList("default"), + null, + "d4a10b5c-1e8a-4b72-9d67-3f4a8c9e1b2d", + 6148331192489823102L, + null); + + RefreshState refreshState = + new RefreshState(1, Collections.singletonList(tableState), 1573518435000L); + + String json = RefreshStateParser.toJson(refreshState); + RefreshState parsed = RefreshStateParser.fromJson(json); + + Assertions.assertThat(parsed.viewVersionId()).isEqualTo(1); + Assertions.assertThat(parsed.refreshStartTimestampMs()).isEqualTo(1573518435000L); + Assertions.assertThat(parsed.sourceStates()).hasSize(1); + + SourceState source = parsed.sourceStates().get(0); + Assertions.assertThat(source).isInstanceOf(SourceTableState.class); + Assertions.assertThat(source.type()).isEqualTo("table"); + Assertions.assertThat(source.name()).isEqualTo("events"); + Assertions.assertThat(source.namespace()).containsExactly("default"); + Assertions.assertThat(source.catalog()).isNull(); + Assertions.assertThat(source.uuid()).isEqualTo("d4a10b5c-1e8a-4b72-9d67-3f4a8c9e1b2d"); + + SourceTableState parsedTable = (SourceTableState) source; + Assertions.assertThat(parsedTable.snapshotId()).isEqualTo(6148331192489823102L); + Assertions.assertThat(parsedTable.ref()).isNull(); + } + + @Test + public void testRoundTripSourceViewState() { + SourceViewState viewState = + new SourceViewState( + "daily_summary", + Arrays.asList("analytics", "views"), + "other_catalog", + "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + 5); + + RefreshState refreshState = + new RefreshState(2, Collections.singletonList(viewState), 1573518435000L); + + String json = RefreshStateParser.toJson(refreshState); + RefreshState parsed = RefreshStateParser.fromJson(json); + + Assertions.assertThat(parsed.sourceStates()).hasSize(1); + + SourceState source = parsed.sourceStates().get(0); + Assertions.assertThat(source).isInstanceOf(SourceViewState.class); + Assertions.assertThat(source.type()).isEqualTo("view"); + Assertions.assertThat(source.name()).isEqualTo("daily_summary"); + Assertions.assertThat(source.namespace()).containsExactly("analytics", "views"); + Assertions.assertThat(source.catalog()).isEqualTo("other_catalog"); + + SourceViewState parsedView = (SourceViewState) source; + Assertions.assertThat(parsedView.versionId()).isEqualTo(5); + } + + @Test + public void testRoundTripMixedSourceStates() { + SourceTableState tableState = + new SourceTableState( + "events", + Arrays.asList("default"), + null, + "uuid-1", + 100L, + "main"); + + SourceViewState viewState = + new SourceViewState( + "event_summary", + Arrays.asList("default"), + null, + "uuid-2", + 3); + + RefreshState refreshState = + new RefreshState(1, Arrays.asList(tableState, viewState), 1573518435000L); + + String json = RefreshStateParser.toJson(refreshState); + RefreshState parsed = RefreshStateParser.fromJson(json); + + Assertions.assertThat(parsed.sourceStates()).hasSize(2); + Assertions.assertThat(parsed.sourceStates().get(0)).isInstanceOf(SourceTableState.class); + Assertions.assertThat(parsed.sourceStates().get(1)).isInstanceOf(SourceViewState.class); + + SourceTableState parsedTable = (SourceTableState) parsed.sourceStates().get(0); + Assertions.assertThat(parsedTable.ref()).isEqualTo("main"); + } + + @Test + public void testParseSpecExample() { + String json = + "{" + + "\"view-version-id\":1," + + "\"refresh-start-timestamp-ms\":1573518435000," + + "\"source-states\":[{" + + "\"type\":\"table\"," + + "\"namespace\":[\"default\"]," + + "\"name\":\"events\"," + + "\"uuid\":\"d4a10b5c-1e8a-4b72-9d67-3f4a8c9e1b2d\"," + + "\"snapshot-id\":6148331192489823102" + + "}]" + + "}"; + + RefreshState parsed = RefreshStateParser.fromJson(json); + + Assertions.assertThat(parsed.viewVersionId()).isEqualTo(1); + Assertions.assertThat(parsed.refreshStartTimestampMs()).isEqualTo(1573518435000L); + Assertions.assertThat(parsed.sourceStates()).hasSize(1); + + SourceTableState tableState = (SourceTableState) parsed.sourceStates().get(0); + Assertions.assertThat(tableState.name()).isEqualTo("events"); + Assertions.assertThat(tableState.namespace()).containsExactly("default"); + Assertions.assertThat(tableState.uuid()) + .isEqualTo("d4a10b5c-1e8a-4b72-9d67-3f4a8c9e1b2d"); + Assertions.assertThat(tableState.snapshotId()).isEqualTo(6148331192489823102L); + } + + @Test + public void testEmptySourceStates() { + RefreshState refreshState = + new RefreshState(1, Collections.emptyList(), 1573518435000L); + + String json = RefreshStateParser.toJson(refreshState); + RefreshState parsed = RefreshStateParser.fromJson(json); + + Assertions.assertThat(parsed.sourceStates()).isEmpty(); + Assertions.assertThat(parsed.viewVersionId()).isEqualTo(1); + } + + @Test + public void testSourceTableStateWithRef() { + SourceTableState tableState = + new SourceTableState( + "events", Arrays.asList("default"), null, "uuid-1", 100L, "audit_branch"); + + RefreshState refreshState = + new RefreshState(1, Collections.singletonList(tableState), 1573518435000L); + + String json = RefreshStateParser.toJson(refreshState); + Assertions.assertThat(json).contains("\"ref\":\"audit_branch\""); + + RefreshState parsed = RefreshStateParser.fromJson(json); + SourceTableState parsedTable = (SourceTableState) parsed.sourceStates().get(0); + Assertions.assertThat(parsedTable.ref()).isEqualTo("audit_branch"); + } + + @Test + public void testNullJsonThrows() { + Assertions.assertThatThrownBy(() -> RefreshStateParser.fromJson((String) null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse refresh state from null string"); + } + + @Test + public void testUnknownTypeThrows() { + String json = + "{" + + "\"view-version-id\":1," + + "\"refresh-start-timestamp-ms\":1573518435000," + + "\"source-states\":[{" + + "\"type\":\"unknown\"," + + "\"namespace\":[\"default\"]," + + "\"name\":\"events\"," + + "\"uuid\":\"uuid-1\"" + + "}]" + + "}"; + + Assertions.assertThatThrownBy(() -> RefreshStateParser.fromJson(json)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unknown source state type: unknown"); + } +} diff --git a/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java b/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java index a46e63401632..d27153a77b00 100644 --- a/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java +++ b/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.JsonNode; import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; @@ -119,6 +120,81 @@ public void testNullViewVersion() { .hasMessage("Cannot parse view version from null string"); } + @Test + public void testViewVersionWithStorageTable() { + SQLViewRepresentation representation = + ImmutableSQLViewRepresentation.builder() + .sql("select * from events") + .dialect("spark") + .build(); + + TableIdentifier storageTable = TableIdentifier.of(Namespace.of("default"), "mv__storage"); + + ViewVersion viewVersion = + ImmutableViewVersion.builder() + .versionId(1) + .timestampMillis(12345) + .addRepresentations(representation) + .summary(ImmutableMap.of("engine-name", "Spark")) + .defaultNamespace(Namespace.of("default")) + .defaultCatalog("prod") + .schemaId(1) + .storageTable(storageTable) + .build(); + + String json = ViewVersionParser.toJson(viewVersion); + Assertions.assertThat(json).contains("\"storage-table\":{"); + Assertions.assertThat(json).contains("\"namespace\":[\"default\"]"); + Assertions.assertThat(json).contains("\"name\":\"mv__storage\""); + + ViewVersion parsed = ViewVersionParser.fromJson(json); + Assertions.assertThat(parsed.storageTable()).isNotNull(); + Assertions.assertThat(parsed.storageTable().namespace()).isEqualTo(Namespace.of("default")); + Assertions.assertThat(parsed.storageTable().name()).isEqualTo("mv__storage"); + Assertions.assertThat(parsed).isEqualTo(viewVersion); + } + + @Test + public void testViewVersionWithoutStorageTable() { + SQLViewRepresentation representation = + ImmutableSQLViewRepresentation.builder() + .sql("select * from events") + .dialect("spark") + .build(); + + ViewVersion viewVersion = + ImmutableViewVersion.builder() + .versionId(1) + .timestampMillis(12345) + .addRepresentations(representation) + .summary(ImmutableMap.of()) + .defaultNamespace(Namespace.of("default")) + .schemaId(1) + .build(); + + String json = ViewVersionParser.toJson(viewVersion); + Assertions.assertThat(json).doesNotContain("storage-table"); + + ViewVersion parsed = ViewVersionParser.fromJson(json); + Assertions.assertThat(parsed.storageTable()).isNull(); + } + + @Test + public void testParseViewVersionWithStorageTableJson() { + String json = + "{\"version-id\":1,\"timestamp-ms\":1573518431292,\"schema-id\":1," + + "\"summary\":{\"engine-name\":\"Spark\",\"engine-version\":\"3.4.1\"}," + + "\"default-catalog\":\"prod\",\"default-namespace\":[\"default\"]," + + "\"representations\":[{\"type\":\"sql\",\"sql\":\"SELECT COUNT(1) FROM events\"," + + "\"dialect\":\"spark\"}]," + + "\"storage-table\":{\"namespace\":[\"default\"],\"name\":\"event_agg_mv__storage\"}}"; + + ViewVersion parsed = ViewVersionParser.fromJson(json); + Assertions.assertThat(parsed.storageTable()).isNotNull(); + Assertions.assertThat(parsed.storageTable().namespace()).isEqualTo(Namespace.of("default")); + Assertions.assertThat(parsed.storageTable().name()).isEqualTo("event_agg_mv__storage"); + } + @Test public void missingDefaultCatalog() { assertThatThrownBy( diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala index 86306e76827e..c13595c08fd1 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala @@ -169,8 +169,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) } private def getCreateMaterializedViewStatement(sqlText: String): String = { - val replace1 = CREATE_MATERIALIZED_VIEW_PATTERN.replaceAllIn(sqlText, m => m.group(1) + " " + m.group(2)) - MATERIALIZED_VIEW_STORED_AS_PATTERN.replaceAllIn(replace1, "") + val createViewSql = CREATE_MATERIALIZED_VIEW_PATTERN.replaceAllIn(sqlText, m => m.group(1) + " " + m.group(2)) + MATERIALIZED_VIEW_STORED_AS_PATTERN.replaceAllIn(createViewSql, "") } private def getMaterializedViewOptions(sqlText: String): MaterializedViewOptions = { diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala index 6ccc63c96f5b..79cc5eb02548 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -20,20 +20,15 @@ package org.apache.spark.sql.execution.datasources.v2 -import org.apache.iceberg.{SnapshotUpdate, catalog} - -import java.util.UUID import org.apache.iceberg.catalog.{Namespace, TableIdentifier} import org.apache.iceberg.relocated.com.google.common.base.Preconditions import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap import org.apache.iceberg.spark.{MaterializedViewUtil, Spark3Util, SparkCatalog, SparkSchemaUtil} -import org.apache.iceberg.spark.source.{SparkTable, SparkView} +import org.apache.iceberg.spark.source.SparkView import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.connector.catalog.Identifier -import org.apache.spark.sql.connector.catalog.Table -import org.apache.spark.sql.connector.catalog.TableChange import org.apache.spark.sql.connector.catalog.View import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.connector.expressions.Transform @@ -75,40 +70,32 @@ case class CreateMaterializedViewExec( case None => MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier(ident) } - val view = createView(sparkStorageTableIdentifier.toString) - - view match { - case Some(v) => { - // TODO: Add support for partitioning the storage table - catalog.asInstanceOf[SparkCatalog].createTable( - sparkStorageTableIdentifier, - viewSchema, new Array[Transform](0), ImmutableMap.of[String, String]() - ) - - // Capture base table state before inserting into the storage table - val baseTables = MaterializedViewUtil.extractBaseTables(queryText).asScala.toList - val baseTableSnapshots = getBaseTableSnapshots(baseTables) - val baseTableSnapshotsProperties = baseTableSnapshots.map { - case (key, value) => ( - MaterializedViewUtil.MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX + key.toString - ) -> value.toString - } - - - val storageTableProperties = baseTableSnapshotsProperties + - (MaterializedViewUtil.MATERIALIZED_VIEW_VERSION_PROPERTY_KEY -> getViewVersion(v).toString) - - // Insert into the storage table - session.sql("INSERT INTO " + sparkStorageTableIdentifier + " " + queryText) - - // Load the storage table as an Iceberg table - val icebergStorageTable = catalog.asInstanceOf[org.apache.iceberg.catalog.Catalog].loadTable(TableIdentifier.parse(sparkStorageTableIdentifier.toString)) - val replaceSnapshot = icebergStorageTable.newRewrite() - replaceSnapshot.set("mv-base-table-snapshots", baseTableSnapshots.asJava.toString) - replaceSnapshot.commit() + // Step 1: Create the storage table BEFORE the MV view metadata. + // Per spec: "The storage table must exist and be accessible before the + // materialized view metadata is committed." + // A newly created MV has a storage table with no snapshots until a refresh is performed. + catalog.asInstanceOf[SparkCatalog].createTable( + sparkStorageTableIdentifier, + viewSchema, new Array[Transform](0), ImmutableMap.of[String, String]() + ) + + // Step 2: Create the MV view metadata with a storage-table reference + try { + createView(sparkStorageTableIdentifier.toString) match { + case Some(_) => // success + case None => // allowExisting and view already exists } - case None => + } catch { + case e: Exception => + // If view creation fails, clean up the storage table + try { + catalog.asInstanceOf[SparkCatalog].dropTable(sparkStorageTableIdentifier) + } catch { + case _: Exception => // best effort cleanup + } + throw e } + Nil } @@ -127,8 +114,6 @@ case class CreateMaterializedViewExec( comment.map(ViewCatalog.PROP_COMMENT -> _) + (ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion, ViewCatalog.PROP_ENGINE_VERSION -> engineVersion) + - (MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY -> "true") + - (MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_TABLE_PROPERTY_KEY -> storageTableIdentifier) + ("queryColumnNames" -> queryColumnNames.mkString(",")) @@ -138,7 +123,7 @@ case class CreateMaterializedViewExec( catalog.dropView(ident) } // FIXME: replaceView API doesn't exist in Spark 3.5 - val icebergView = catalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( + val icebergView = catalog.asInstanceOf[SparkCatalog].icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( Spark3Util.identifierToTableIdentifier(ident)) .withDefaultCatalog(currentCatalog) .withDefaultNamespace(Namespace.of(currentNamespace: _*)) @@ -153,7 +138,7 @@ case class CreateMaterializedViewExec( } else { try { // CREATE VIEW [IF NOT EXISTS] - val icebergView = catalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( + val icebergView = catalog.asInstanceOf[SparkCatalog].icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( Spark3Util.identifierToTableIdentifier(ident)) .withDefaultCatalog(currentCatalog) .withDefaultNamespace(Namespace.of(currentNamespace: _*)) @@ -171,23 +156,4 @@ case class CreateMaterializedViewExec( } } - private def getBaseTableSnapshots(baseTables: List[Table]): Map[UUID, Long] = { - baseTables.map { - case sparkTable: SparkTable => - val snapshot = Option(sparkTable.table().currentSnapshot()) - val snapshotId = snapshot.map(_.snapshotId().longValue()).getOrElse(0L) - (sparkTable.table().uuid(), snapshotId) - case _ => - throw new UnsupportedOperationException("Only Spark tables are supported") - }.toMap - } - - private def getViewVersion(view: View): Long = { - view match { - case sparkView: SparkView => - sparkView.view().currentVersion().versionId() - case _ => - throw new UnsupportedOperationException("Only Spark views are supported") - } - } } diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala index b57473c36383..577a588e66a1 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala @@ -21,11 +21,8 @@ package org.apache.spark.sql.execution.datasources.v2 import org.apache.iceberg.catalog.Namespace import org.apache.iceberg.catalog.TableIdentifier import org.apache.iceberg.exceptions -import org.apache.iceberg.spark.MaterializedViewUtil -import org.apache.iceberg.spark.Spark3Util import org.apache.iceberg.spark.SparkCatalog import org.apache.iceberg.view.View -import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.NoSuchViewException import org.apache.spark.sql.catalyst.expressions.Attribute @@ -50,23 +47,14 @@ case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boo } } } - // if view is not null read the properties and check if it is a materialized view + // if view is a materialized view, drop the storage table first view match { - case Some(v) => { - val viewProperties = v.properties(); - if (Option( - viewProperties.get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY - )).getOrElse("false").equals("true")) { - // get the storage table location then drop the storage table - val storageTableLocation = viewProperties.get( - MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_TABLE_PROPERTY_KEY - ) - val storageTableIdentifier = Spark3Util.catalogAndIdentifier( - SparkSession.active, storageTableLocation).identifier() - // get active spark session - catalog.asInstanceOf[SparkCatalog].dropTable(storageTableIdentifier) + case Some(v) => + val storageTable = v.currentVersion().storageTable() + if (storageTable != null) { + val storageIdent = Identifier.of(storageTable.namespace().levels(), storageTable.name()) + catalog.asInstanceOf[SparkCatalog].dropTable(storageIdent) } - } case _ => } diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index f71fbce9a35c..a05eddfaece3 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.util.Arrays; import java.util.Map; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.TableOperations; @@ -37,9 +38,14 @@ import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.spark.MaterializedViewUtil; +import org.apache.iceberg.spark.SparkCatalog; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.SparkMaterializedView; import org.apache.iceberg.spark.source.SparkView; +import org.apache.iceberg.view.RefreshState; +import org.apache.iceberg.view.RefreshStateParser; +import org.apache.iceberg.view.SourceTableState; +import org.apache.iceberg.view.View; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; import org.apache.spark.sql.connector.catalog.CatalogPlugin; @@ -89,7 +95,7 @@ public static Object[][] parameters() { private static String getTempWarehouseDir() { try { File tempDir = Files.createTempDirectory("warehouse-").toFile(); - tempDir.delete(); + tempDir.deleteOnExit(); return tempDir.getAbsolutePath(); } catch (IOException e) { @@ -103,65 +109,95 @@ public TestMaterializedViews( } @Test - public void assertReadFromStorageTableWhenFresh() throws IOException { - sql("DROP VIEW IF EXISTS %s", materializedViewName); + public void testStorageTableFieldOnViewVersion() { sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - // Assert that number of records in the materialized view is the same as the number of records - // in the table - assertThat(sql("SELECT * FROM %s", materializedViewName).size()) - .isEqualTo(sql("SELECT * FROM %s", tableName).size()); + View view = loadIcebergView(); + // storage-table should be set on the view version, not as a property + assertThat(view.currentVersion().storageTable()).isNotNull(); + assertThat(view.currentVersion().storageTable().name()) + .isEqualTo(materializedViewName + "__storage"); + assertThat(view.currentVersion().storageTable().namespace()) + .isEqualTo(NAMESPACE); + } - // Assert that the catalog loadView method throws IllegalStateException because the view is - // fresh - assertThatThrownBy(() -> sparkViewCatalog().loadView(viewIdentifier())) - .isInstanceOf(IllegalStateException.class); + @Test + public void testNeverRefreshedMvIsNotFresh() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - // Assert that the catalog loadTable method returns an object, and its type is - // SparkMaterializedView + // A newly created MV has no snapshots on its storage table, so it's not fresh. + // loadView should succeed (returns stale view) + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); + } catch (NoSuchViewException e) { + fail("Materialized view not found"); + } + } + + @Test + public void testReadFromStorageTableWhenFresh() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + simulateRefresh(); + + // Fresh MV: loadTable should return SparkMaterializedView try { assertThat(sparkTableCatalog().loadTable(viewIdentifier())) .isInstanceOf(SparkMaterializedView.class); } catch (NoSuchTableException e) { - fail("Materialized view storage table not found"); + fail("Fresh materialized view should be loadable as a table"); } + + // Fresh MV: loadView should throw since the engine should use loadTable instead + assertThatThrownBy(() -> sparkViewCatalog().loadView(viewIdentifier())) + .isInstanceOf(IllegalStateException.class); } @Test - public void assertNotReadFromStorageTableWhenStale() throws IOException { + public void testFallbackToViewWhenStale() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - // Insert one row to the table so the materialized view becomes stale - sql("INSERT INTO %s VALUES (1, 'a')", tableName); + simulateRefresh(); - // Assert that number of records in the materialized view is the same as the number of records - // in the table - assertThat(sql("SELECT * FROM %s", materializedViewName).size()) - .isEqualTo(sql("SELECT * FROM %s", tableName).size()); + // Insert more data to invalidate the refresh + sql("INSERT INTO %s VALUES (3, 'c')", tableName); - // Assert that the catalog loadView method returns an object, and of type SparkView + // Stale MV: loadView should return SparkView (falls back to query execution) try { - assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); + assertThat(sparkViewCatalog().loadView(viewIdentifier())) + .isInstanceOf(SparkView.class); } catch (NoSuchViewException e) { - fail("Materialized view not found"); + fail("Stale materialized view should be loadable as a view"); } - // Assert that the catalog loadTable fails with NoSuchTableException because the view is stale + // Stale MV: loadTable should not resolve to the MV's storage table assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) .isInstanceOf(NoSuchTableException.class); } @Test - public void testDefaultStorageTableIdentifier() { + public void testStorageTableCreatedBeforeMvMetadata() { sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - // Assert that the storage table is in the list of tables - final String materializedViewStorageTableName = + // The storage table should exist + String storageTableName = MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier( Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName)) .name(); assertThat(sql("SHOW TABLES")) - .anySatisfy(row -> assertThat(row[1]).isEqualTo(materializedViewStorageTableName)); + .anySatisfy(row -> assertThat(row[1]).isEqualTo(storageTableName)); + } + + @Test + public void testDefaultStorageTableNaming() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // Default naming should be __storage + String expectedStorageTableName = materializedViewName + "__storage"; + assertThat(sql("SHOW TABLES")) + .anySatisfy(row -> assertThat(row[1]).isEqualTo(expectedStorageTableName)); } @Test @@ -175,6 +211,49 @@ public void testStoredAsClause() { assertThat(sql("SHOW TABLES")).anySatisfy(row -> assertThat(row[1]).isEqualTo(customTableName)); } + private void simulateRefresh() { + View view = loadIcebergView(); + org.apache.iceberg.catalog.TableIdentifier storageTableId = + view.currentVersion().storageTable(); + + // Get the base table's current snapshot ID + long baseSnapshotId = + (Long) + sql( + "SELECT snapshot_id FROM %s.%s.%s.snapshots ORDER BY committed_at DESC LIMIT 1", + catalogName, NAMESPACE, tableName) + .get(0)[0]; + + // Build refresh state matching the current view version and source table state + RefreshState refreshState = + new RefreshState( + view.currentVersion().versionId(), + Arrays.asList( + new SourceTableState( + tableName, + Arrays.asList(NAMESPACE.levels()), + null, + "test-uuid", + baseSnapshotId, + null)), + System.currentTimeMillis()); + String refreshStateJson = RefreshStateParser.toJson(refreshState); + + // Write data to storage table with refresh-state in the snapshot summary + String storageTableRef = + String.format("%s.%s.%s", catalogName, NAMESPACE, storageTableId.name()); + try { + spark + .sql(String.format("SELECT id, data FROM %s.%s.%s", catalogName, NAMESPACE, tableName)) + .writeTo(storageTableRef) + .option( + "snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) + .append(); + } catch (NoSuchTableException e) { + throw new RuntimeException("Storage table not found during simulated refresh", e); + } + } + private ViewCatalog sparkViewCatalog() { CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); return (ViewCatalog) catalogPlugin; @@ -189,6 +268,17 @@ private Identifier viewIdentifier() { return Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName); } + private SparkCatalog sparkCatalog() { + return (SparkCatalog) spark.sessionState().catalogManager().catalog(catalogName); + } + + private View loadIcebergView() { + org.apache.iceberg.catalog.ViewCatalog icebergViewCatalog = + (org.apache.iceberg.catalog.ViewCatalog) sparkCatalog().icebergCatalog(); + return icebergViewCatalog.loadView( + TableIdentifier.of(NAMESPACE, materializedViewName)); + } + // Required to be public since it is loaded by org.apache.iceberg.CatalogUtil.loadCatalog public static class InMemoryCatalogWithLocalFileIO extends InMemoryCatalog { private FileIO localFileIO; @@ -212,25 +302,30 @@ protected InMemoryCatalog.InMemoryViewOperations newViewOps(TableIdentifier iden private static class LocalFileIO implements FileIO { + private static String stripFilePrefix(String path) { + return path.startsWith("file:") ? path.substring(5) : path; + } + @Override public InputFile newInputFile(String path) { - return org.apache.iceberg.Files.localInput(path); + return org.apache.iceberg.Files.localInput(stripFilePrefix(path)); } @Override public OutputFile newOutputFile(String path) { - return org.apache.iceberg.Files.localOutput(path); + String stripped = stripFilePrefix(path); + java.io.File parent = new java.io.File(stripped).getParentFile(); + if (!parent.isDirectory()) { + parent.mkdirs(); + } + return org.apache.iceberg.Files.localOutput(stripped); } @Override public void deleteFile(String path) { - if (!new File(path).delete()) { + if (!new File(stripFilePrefix(path)).delete()) { throw new RuntimeIOException("Failed to delete file: " + path); } } } - - // TODO Add DROP MATERIALIZED VIEW test - // TODO Assert materialized view creation fails when the location is not provided - // TODO Test cannot replace a materialized view with a new version } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java index 633b705eb37a..a30c5f671176 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java @@ -18,81 +18,13 @@ */ package org.apache.iceberg.spark; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation; -import org.apache.spark.sql.catalyst.parser.ParseException; -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; import org.apache.spark.sql.connector.catalog.Identifier; -import org.apache.spark.sql.connector.catalog.Table; -import org.apache.spark.sql.connector.catalog.TableCatalog; -import scala.collection.JavaConverters; -// Possible to merge with Spark3Util public class MaterializedViewUtil { private MaterializedViewUtil() {} - public static final String MATERIALIZED_VIEW_PROPERTY_KEY = "iceberg.materialized.view"; - public static final String MATERIALIZED_VIEW_STORAGE_TABLE_PROPERTY_KEY = - "iceberg.materialized.view.storage.table"; - public static final String MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX = - "iceberg.base.snapshot."; - public static final String MATERIALIZED_VIEW_VERSION_PROPERTY_KEY = - "iceberg.materialized.view.version"; - private static final String MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX = ".storage.table"; - - public static List

extractBaseTables(String query) { - return extractBaseTableIdentifiers(query).stream() - .filter(optional -> !optional.isEmpty()) - .map(id -> toSparkTable(id).get()) - .collect(Collectors.toList()); - } - - private static List> extractBaseTableIdentifiers(String query) { - try { - // Parse the SQL query to get the LogicalPlan - LogicalPlan logicalPlan = SparkSession.active().sessionState().sqlParser().parsePlan(query); - - // Recursively traverse the LogicalPlan to extract base table names - return extractBaseTableIdentifiers(logicalPlan).stream() - .distinct() - .collect(Collectors.toList()); - } catch (ParseException e) { - throw new IllegalArgumentException("Failed to parse the SQL query: " + query, e); - } - } - - private static List> extractBaseTableIdentifiers(LogicalPlan plan) { - if (plan instanceof UnresolvedRelation) { - UnresolvedRelation relation = (UnresolvedRelation) plan; - List> result = Lists.newArrayListWithCapacity(1); - result.add(JavaConverters.seqAsJavaList(relation.multipartIdentifier())); - return result; - } else { - return (JavaConverters.seqAsJavaList(plan.children())) - .stream() - .flatMap(child -> extractBaseTableIdentifiers(child).stream()) - .collect(Collectors.toList()); - } - } - - public static Optional
toSparkTable(List multipartIdent) { - Spark3Util.CatalogAndIdentifier catalogAndIdentifier = - Spark3Util.catalogAndIdentifier(SparkSession.active(), multipartIdent); - if (catalogAndIdentifier.catalog() instanceof TableCatalog) { - TableCatalog tableCatalog = (TableCatalog) catalogAndIdentifier.catalog(); - try { - return Optional.of(tableCatalog.loadTable(catalogAndIdentifier.identifier())); - } catch (Exception e) { - return Optional.empty(); - } - } - return Optional.empty(); - } + private static final String MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX = "__storage"; public static Identifier getDefaultMaterializedViewStorageTableIdentifier( Identifier viewIdentifier) { diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index 0d68e138bd04..6dcdd074ffb6 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -25,13 +25,11 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CachingCatalog; @@ -80,7 +78,6 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException; import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException; -import org.apache.spark.sql.catalyst.parser.ParseException; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.NamespaceChange; import org.apache.spark.sql.connector.catalog.StagedTable; @@ -613,86 +610,78 @@ public View loadView(Identifier ident) throws NoSuchViewException { throw new NoSuchViewException(ident); } - // Candidate to be moved to org.apache.iceberg.view.View private boolean isMaterializedView(org.apache.iceberg.view.View view) { - return Optional.of(view.properties().get(MaterializedViewUtil.MATERIALIZED_VIEW_PROPERTY_KEY)) - .orElse("false") - .equals("true"); + return view.currentVersion().storageTable() != null; } - // Candidate to be moved to org.apache.iceberg.view.View - private String getStorageTableIdentifier(org.apache.iceberg.view.View view) { - String identifier = - view.properties().get(MaterializedViewUtil.MATERIALIZED_VIEW_STORAGE_TABLE_PROPERTY_KEY); + private org.apache.iceberg.catalog.TableIdentifier getStorageTableId( + org.apache.iceberg.view.View view) { + org.apache.iceberg.catalog.TableIdentifier storageTable = + view.currentVersion().storageTable(); Preconditions.checkState( - identifier != null, "Storage table identifier is not set for materialized view."); - return identifier; + storageTable != null, "Storage table identifier is not set for materialized view."); + return storageTable; } - // Candidate to be moved to org.apache.iceberg.view.View but requires loadTable - private org.apache.iceberg.Table loadStorageTable(org.apache.iceberg.view.View view) { - String storageTableIdentifier = vie - String storageTableIdentifier = getStorageTableIdentifier(view); + private Table loadStorageTable(org.apache.iceberg.view.View view) { + org.apache.iceberg.catalog.TableIdentifier storageTableId = getStorageTableId(view); try { - SparkSession session = SparkSession.active(); - Table storageTable = - loadTable(Spark3Util.catalogAndIdentifier(session, storageTableIdentifier).identifier()); - return storageTable; - } catch (ParseException | NoSuchTableException e) { + Identifier sparkIdent = + Identifier.of(storageTableId.namespace().levels(), storageTableId.name()); + return loadTable(sparkIdent); + } catch (NoSuchTableException e) { throw new IllegalStateException("Unable to load storage table for materialized view.", e); } } - // Candidate to be moved to org.apache.iceberg.view.View but requires loadTable - // Second option is to move to SparkMaterializedView private boolean isFresh(org.apache.iceberg.view.View view) { - Table storageTable = loadStorageTable(view); - Map storageTableProperties = storageTable.properties(); - - // Get the parent view version id from the storage table properties - String storageTableViewVersionIdPropertyValue = - storageTableProperties.get( - MaterializedViewUtil.MATERIALIZED_VIEW_VERSION_PROPERTY_KEY); - if (storageTableViewVersionIdPropertyValue == null) { - throw new IllegalStateException( - "Storage table properties do not contain the virtual view version id property."); + Table sparkStorageTable = loadStorageTable(view); + org.apache.iceberg.Table storageTable = ((SparkTable) sparkStorageTable).table(); + if (storageTable.currentSnapshot() == null) { + return false; + } + + String refreshStateJson = + storageTable + .currentSnapshot() + .summary() + .get(org.apache.iceberg.view.RefreshState.REFRESH_STATE_SUMMARY_KEY); + if (refreshStateJson == null) { + return false; } - int storageTableViewVersionId = Integer.parseInt(storageTableViewVersionIdPropertyValue); - // If the storage table view version id is different from the current version id, the - // materialized view is not fresh - if (storageTableViewVersionId != view.currentVersion().versionId()) { + org.apache.iceberg.view.RefreshState refreshState = + org.apache.iceberg.view.RefreshStateParser.fromJson(refreshStateJson); + + // If the refresh was performed against a different view version, the MV is not fresh + if (refreshState.viewVersionId() != view.currentVersion().versionId()) { return false; } - // Get the base table snapshot ids from the storage table properties - Map baseTableSnapshotsProperties = - storageTableProperties.entrySet().stream() - .filter( - entry -> - entry - .getKey() - .startsWith( - MaterializedViewUtil - .MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - List
baseTables = MaterializedViewUtil.extractBaseTables(view.sqlFor("spark").sql()); - - for (Table baseTable : baseTables) { - org.apache.iceberg.Table icebergBaseTable = ((SparkTable) baseTable).table(); - String snapshotId = - String.valueOf( - icebergBaseTable.currentSnapshot() == null - ? 0 - : icebergBaseTable.currentSnapshot().snapshotId()); - if (!baseTableSnapshotsProperties - .get( - MaterializedViewUtil.MATERIALIZED_VIEW_BASE_SNAPSHOT_PROPERTY_KEY_PREFIX - + icebergBaseTable.uuid()) - .equals(snapshotId)) { - return false; + // Check each source table state against the current state + for (org.apache.iceberg.view.SourceState sourceState : refreshState.sourceStates()) { + if (sourceState instanceof org.apache.iceberg.view.SourceTableState) { + org.apache.iceberg.view.SourceTableState tableState = + (org.apache.iceberg.view.SourceTableState) sourceState; + org.apache.iceberg.catalog.TableIdentifier sourceId = + org.apache.iceberg.catalog.TableIdentifier.of( + org.apache.iceberg.catalog.Namespace.of( + tableState.namespace().toArray(new String[0])), + tableState.name()); + try { + org.apache.iceberg.Table sourceTable = + ((org.apache.iceberg.catalog.Catalog) icebergCatalog()).loadTable(sourceId); + long currentSnapshotId = + sourceTable.currentSnapshot() == null ? -1 : sourceTable.currentSnapshot().snapshotId(); + if (currentSnapshotId != tableState.snapshotId()) { + return false; + } + } catch (Exception e) { + return false; + } } } + return true; } From cc3bff4683b1ffacd2212e6562fb02bb91a4a9a3 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 24 Mar 2026 11:19:04 -0700 Subject: [PATCH 06/22] Port MV support to Spark 4.1; fix DropV2ViewExec cast --- .../org/apache/iceberg/view/ViewBuilder.java | 11 +- .../iceberg/inmemory/InMemoryCatalog.java | 6 +- .../iceberg/view/TestViewVersionParser.java | 24 +- .../v2/CreateOrReplaceTagExec.scala | 3 +- .../datasources/v2/DropV2ViewExec.scala | 40 +- .../sql/catalyst/analysis/CheckViews.scala | 1 + .../sql/catalyst/analysis/ResolveViews.scala | 1 + .../analysis/RewriteViewCommands.scala | 7 +- .../IcebergSparkSqlExtensionsParser.scala | 22 ++ .../logical/views/CreateIcebergView.scala | 2 + .../v2/CreateMaterializedViewExec.scala | 159 ++++++++ .../datasources/v2/DropV2ViewExec.scala | 30 ++ .../v2/ExtendedDataSourceV2Strategy.scala | 29 ++ .../extensions/TestMaterializedViews.java | 341 ++++++++++++++++++ .../iceberg/spark/MaterializedViewUtil.java | 35 ++ .../apache/iceberg/spark/SparkCatalog.java | 96 ++++- .../spark/source/SparkMaterializedView.java | 57 +++ .../iceberg/spark/SparkCatalogConfig.java | 12 +- 18 files changed, 831 insertions(+), 45 deletions(-) create mode 100644 spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala create mode 100644 spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java create mode 100644 spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java create mode 100644 spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java diff --git a/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java b/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java index 4809878ba958..5e025f19bb3a 100644 --- a/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java +++ b/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java @@ -19,7 +19,6 @@ package org.apache.iceberg.view; import java.util.Map; - import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.catalog.ViewCatalog; @@ -57,12 +56,16 @@ default ViewBuilder withLocation(String location) { throw new UnsupportedOperationException("Setting a view's location is not supported"); } - /** Set the storage table identifier in case of a materialized view. + /** + * Sets the storage table identifier for a materialized view. * * @param storageTableIdentifier the storage table identifier * @return this for method chaining - */ - ViewBuilder withStorageTableIdentifier(TableIdentifier storageTableIdentifier); + */ + default ViewBuilder withStorageTableIdentifier(TableIdentifier storageTableIdentifier) { + throw new UnsupportedOperationException( + "Setting a storage table identifier is not supported"); + } /** * Create the view. diff --git a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java index 74730eb37e25..c400f2029b42 100644 --- a/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java +++ b/core/src/main/java/org/apache/iceberg/inmemory/InMemoryCatalog.java @@ -106,7 +106,6 @@ public void initialize(String name, Map properties) { closeableGroup.setSuppressCloseFailure(true); } - // protected for testing @Override protected TableOperations newTableOps(TableIdentifier tableIdentifier) { return new InMemoryTableOperations(io, tableIdentifier); @@ -355,7 +354,6 @@ public List listViews(Namespace namespace) { .collect(Collectors.toList()); } - // protected for testing @Override protected ViewOperations newViewOps(TableIdentifier identifier) { return new InMemoryViewOperations(io, identifier); @@ -403,7 +401,7 @@ protected Map properties() { return catalogProperties == null ? ImmutableMap.of() : catalogProperties; } - public class InMemoryTableOperations extends BaseMetastoreTableOperations { + protected class InMemoryTableOperations extends BaseMetastoreTableOperations { private final FileIO fileIO; private final TableIdentifier tableIdentifier; private final String fullTableName; @@ -474,7 +472,7 @@ protected String tableName() { } } - public class InMemoryViewOperations extends BaseViewOperations { + protected class InMemoryViewOperations extends BaseViewOperations { private final FileIO io; private final TableIdentifier identifier; private final String fullViewName; diff --git a/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java b/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java index d27153a77b00..c501d0247724 100644 --- a/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java +++ b/core/src/test/java/org/apache/iceberg/view/TestViewVersionParser.java @@ -143,15 +143,15 @@ public void testViewVersionWithStorageTable() { .build(); String json = ViewVersionParser.toJson(viewVersion); - Assertions.assertThat(json).contains("\"storage-table\":{"); - Assertions.assertThat(json).contains("\"namespace\":[\"default\"]"); - Assertions.assertThat(json).contains("\"name\":\"mv__storage\""); + assertThat(json).contains("\"storage-table\":{"); + assertThat(json).contains("\"namespace\":[\"default\"]"); + assertThat(json).contains("\"name\":\"mv__storage\""); ViewVersion parsed = ViewVersionParser.fromJson(json); - Assertions.assertThat(parsed.storageTable()).isNotNull(); - Assertions.assertThat(parsed.storageTable().namespace()).isEqualTo(Namespace.of("default")); - Assertions.assertThat(parsed.storageTable().name()).isEqualTo("mv__storage"); - Assertions.assertThat(parsed).isEqualTo(viewVersion); + assertThat(parsed.storageTable()).isNotNull(); + assertThat(parsed.storageTable().namespace()).isEqualTo(Namespace.of("default")); + assertThat(parsed.storageTable().name()).isEqualTo("mv__storage"); + assertThat(parsed).isEqualTo(viewVersion); } @Test @@ -173,10 +173,10 @@ public void testViewVersionWithoutStorageTable() { .build(); String json = ViewVersionParser.toJson(viewVersion); - Assertions.assertThat(json).doesNotContain("storage-table"); + assertThat(json).doesNotContain("storage-table"); ViewVersion parsed = ViewVersionParser.fromJson(json); - Assertions.assertThat(parsed.storageTable()).isNull(); + assertThat(parsed.storageTable()).isNull(); } @Test @@ -190,9 +190,9 @@ public void testParseViewVersionWithStorageTableJson() { + "\"storage-table\":{\"namespace\":[\"default\"],\"name\":\"event_agg_mv__storage\"}}"; ViewVersion parsed = ViewVersionParser.fromJson(json); - Assertions.assertThat(parsed.storageTable()).isNotNull(); - Assertions.assertThat(parsed.storageTable().namespace()).isEqualTo(Namespace.of("default")); - Assertions.assertThat(parsed.storageTable().name()).isEqualTo("event_agg_mv__storage"); + assertThat(parsed.storageTable()).isNotNull(); + assertThat(parsed.storageTable().namespace()).isEqualTo(Namespace.of("default")); + assertThat(parsed.storageTable().name()).isEqualTo("event_agg_mv__storage"); } @Test diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala index 03c7e1385fff..e486892614cb 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateOrReplaceTagExec.scala @@ -40,8 +40,7 @@ case class CreateOrReplaceTagExec( override lazy val output: Seq[Attribute] = Nil override protected def run(): Seq[InternalRow] = { - catalog - .loadTable(ident) match { + catalog.loadTable(ident) match { case iceberg: SparkTable => val snapshotId: java.lang.Long = tagOptions.snapshotId .orElse(Option(iceberg.table.currentSnapshot()).map(_.snapshotId())) diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala index 577a588e66a1..a5f5285131c7 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala @@ -35,27 +35,29 @@ case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boo override lazy val output: Seq[Attribute] = Nil override protected def run(): Seq[InternalRow] = { - val icebergCatalog = catalog.asInstanceOf[SparkCatalog].icebergCatalog() - val icebergViewCatalog = icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - var view: Option[View] = None - try { - view = Some(icebergViewCatalog.loadView(TableIdentifier.of(Namespace.of(ident.namespace(): _*), ident.name()))) - } catch { - case e: exceptions.NoSuchViewException => { - if (!ifExists) { - throw new NoSuchViewException(ident) + // If the catalog is a SparkCatalog, check for materialized view storage table cleanup + catalog match { + case sparkCatalog: SparkCatalog => + val icebergCatalog = sparkCatalog.icebergCatalog() + val icebergViewCatalog = icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] + var view: Option[View] = None + try { + view = Some(icebergViewCatalog.loadView(TableIdentifier.of(Namespace.of(ident.namespace(): _*), ident.name()))) + } catch { + case _: exceptions.NoSuchViewException => + if (!ifExists) { + throw new NoSuchViewException(ident) + } } - } - } - // if view is a materialized view, drop the storage table first - view match { - case Some(v) => - val storageTable = v.currentVersion().storageTable() - if (storageTable != null) { - val storageIdent = Identifier.of(storageTable.namespace().levels(), storageTable.name()) - catalog.asInstanceOf[SparkCatalog].dropTable(storageIdent) + // if view is a materialized view, drop the storage table first + view.foreach { v => + val storageTable = v.currentVersion().storageTable() + if (storageTable != null) { + val storageIdent = Identifier.of(storageTable.namespace().levels(), storageTable.name()) + sparkCatalog.dropTable(storageIdent) + } } - case _ => + case _ => // not a SparkCatalog, skip MV cleanup } val dropped = catalog.dropView(ident) diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala index 5ad4b9c01409..1ad164752615 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala @@ -49,6 +49,7 @@ object CheckViews extends (LogicalPlan => Unit) { _, replace, _, + _, _) => verifyColumnCount(resolvedIdent, columnAliases, query) SchemaUtils.checkColumnNameDuplication( diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala index 83e501257ced..4f8d5335674f 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala @@ -76,6 +76,7 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look _, _, _, + _, _) if query.resolved && !c.rewritten => val aliased = aliasColumns(query, columnAliases, columnComments) c.copy( diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala index ac0f75c422d1..c6b988255cc6 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala @@ -40,7 +40,7 @@ import scala.collection.mutable * ResolveSessionCatalog exits early for some v2 View commands, * thus they are pre-substituted here and then handled in ResolveViews */ -case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog { +case class RewriteViewCommands(spark: SparkSession, materializedViewOptions: Option[MaterializedViewOptions] = None) extends Rule[LogicalPlan] with LookupCatalog { import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ @@ -72,7 +72,8 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi comment = comment, properties = properties, allowExisting = allowExisting, - replace = replace) + replace = replace, + materializedViewOptions = materializedViewOptions) case view @ ShowViews(CurrentNamespace, pattern, output) => if (ViewUtil.isViewCatalog(catalogManager.currentCatalog)) { @@ -208,3 +209,5 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi tempFunctions.toSeq } } + +case class MaterializedViewOptions(storageTableIdentifier: Option[String]) diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala index 7c737f0513ed..e74eceddfc05 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.MaterializedViewOptions import org.apache.spark.sql.catalyst.analysis.RewriteViewCommands import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.ParameterContext @@ -53,6 +54,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) private lazy val substitutor = substitutorCtor.newInstance(SQLConf.get) private lazy val astBuilder = new IcebergSqlExtensionsAstBuilder(delegate) + private lazy final val CREATE_MATERIALIZED_VIEW_PATTERN = "(?i)(CREATE)\\s+MATERIALIZED\\s+(VIEW)".r + private lazy final val MATERIALIZED_VIEW_STORED_AS_PATTERN = "(?i)STORED AS\\s*'(\\w+)'\\s*".r /** * Parse a string to a DataType. @@ -142,11 +145,30 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) if (isIcebergCommand(sqlTextAfterSubstitution)) { parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) } .asInstanceOf[LogicalPlan] + } else if (isCreateMaterializedView(sqlText)) { + RewriteViewCommands(SparkSession.active, Option(getMaterializedViewOptions(sqlText))).apply( + delegate.parsePlan(getCreateMaterializedViewStatement(sqlText)) + ) } else { RewriteViewCommands(SparkSession.active).apply(delegateParse(sqlText)) } } + private def isCreateMaterializedView(sqlText: String): Boolean = { + sqlText.toLowerCase.contains("create materialized view") + } + + private def getCreateMaterializedViewStatement(sqlText: String): String = { + val createViewSql = CREATE_MATERIALIZED_VIEW_PATTERN.replaceAllIn(sqlText, m => m.group(1) + " " + m.group(2)) + MATERIALIZED_VIEW_STORED_AS_PATTERN.replaceAllIn(createViewSql, "") + } + + private def getMaterializedViewOptions(sqlText: String): MaterializedViewOptions = { + val storedAsPattern = "(?i)STORED AS\\s*'(\\w+)'\\s*".r + val storageTableIdentifier = storedAsPattern.findFirstMatchIn(sqlText).map(_.group(1)) + MaterializedViewOptions(storageTableIdentifier) + } + private def isIcebergCommand(sqlText: String): Boolean = { val normalized = sqlText .toLowerCase(Locale.ROOT) diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala index 84a00a4a9a88..3e11d18b45fe 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.plans.logical.views import org.apache.spark.sql.catalyst.analysis.AnalysisContext +import org.apache.spark.sql.catalyst.analysis.MaterializedViewOptions import org.apache.spark.sql.catalyst.plans.logical.AnalysisOnlyCommand import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -36,6 +37,7 @@ case class CreateIcebergView( allowExisting: Boolean, replace: Boolean, rewritten: Boolean = false, + materializedViewOptions: Option[MaterializedViewOptions] = None, isAnalyzed: Boolean = false) extends AnalysisOnlyCommand { diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala new file mode 100644 index 000000000000..3e27ff78acbb --- /dev/null +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.execution.datasources.v2 + + +import org.apache.iceberg.catalog.{Namespace, TableIdentifier} +import org.apache.iceberg.relocated.com.google.common.base.Preconditions +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap +import org.apache.iceberg.spark.{MaterializedViewUtil, Spark3Util, SparkCatalog, SparkSchemaUtil} +import org.apache.iceberg.spark.source.SparkView +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.View +import org.apache.spark.sql.connector.catalog.ViewCatalog +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.types.StructType + +import scala.jdk.CollectionConverters._ + +case class CreateMaterializedViewExec( + catalog: ViewCatalog, + ident: Identifier, + queryText: String, + viewSchema: StructType, + columnAliases: Seq[String], + columnComments: Seq[Option[String]], + queryColumnNames: Seq[String], + comment: Option[String], + properties: Map[String, String], + allowExisting: Boolean, + replace: Boolean, + storageTableIdentifier: Option[String]) extends LeafV2CommandExec { + + override def output: Seq[Attribute] = Nil + + override protected def run(): Seq[InternalRow] = { + + // Check if storageTableIdentifier is provided. If not, generate a default identifier. + val sparkStorageTableIdentifier = storageTableIdentifier match { + case Some(identifier) => { + val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(session, identifier) + val storageTableCatalogName = catalogAndIdentifier.catalog().name() + Preconditions.checkState( + storageTableCatalogName.equals(catalog.name()), + "Storage table identifier must be in the same catalog as the view." + + " Found storage table in catalog: %s, expected: %s.", + Array[Object](storageTableCatalogName, catalog.name()) + ) + catalogAndIdentifier.identifier() + } + case None => MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier(ident) + } + + // Step 1: Create the storage table BEFORE the MV view metadata. + // Per spec: "The storage table must exist and be accessible before the + // materialized view metadata is committed." + // A newly created MV has a storage table with no snapshots until a refresh is performed. + catalog.asInstanceOf[SparkCatalog].createTable( + sparkStorageTableIdentifier, + viewSchema, new Array[Transform](0), ImmutableMap.of[String, String]() + ) + + // Step 2: Create the MV view metadata with a storage-table reference + try { + createView(sparkStorageTableIdentifier.toString) match { + case Some(_) => // success + case None => // allowExisting and view already exists + } + } catch { + case e: Exception => + // If view creation fails, clean up the storage table + try { + catalog.asInstanceOf[SparkCatalog].dropTable(sparkStorageTableIdentifier) + } catch { + case _: Exception => // best effort cleanup + } + throw e + } + + Nil + } + + override def simpleString(maxFields: Int): String = { + s"CreateMaterializedViewExec: ${ident}" + } + + private def createView(storageTableIdentifier: String): Option[View] = { + val icebergSchema = SparkSchemaUtil.convert(viewSchema) + val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name + val currentCatalog = if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null + val currentNamespace = session.sessionState.catalogManager.currentNamespace + + val engineVersion = "Spark " + org.apache.spark.SPARK_VERSION + val newProperties = properties ++ + comment.map(ViewCatalog.PROP_COMMENT -> _) + + (ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion, + ViewCatalog.PROP_ENGINE_VERSION -> engineVersion) + + ("queryColumnNames" -> queryColumnNames.mkString(",")) + + + if (replace) { + // CREATE OR REPLACE VIEW + if (catalog.viewExists(ident)) { + catalog.dropView(ident) + } + // FIXME: replaceView API doesn't exist in Spark 3.5 + val icebergView = catalog.asInstanceOf[SparkCatalog].icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( + Spark3Util.identifierToTableIdentifier(ident)) + .withDefaultCatalog(currentCatalog) + .withDefaultNamespace(Namespace.of(currentNamespace: _*)) + .withQuery("spark", queryText) + .withSchema(icebergSchema) + .withLocation(properties.get("location").orNull) + .withProperties(newProperties.asJava) + .withStorageTableIdentifier(TableIdentifier.parse(storageTableIdentifier)) + .create() + Some(new SparkView(catalog.name(), icebergView)) + + } else { + try { + // CREATE VIEW [IF NOT EXISTS] + val icebergView = catalog.asInstanceOf[SparkCatalog].icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( + Spark3Util.identifierToTableIdentifier(ident)) + .withDefaultCatalog(currentCatalog) + .withDefaultNamespace(Namespace.of(currentNamespace: _*)) + .withQuery("spark", queryText) + .withSchema(icebergSchema) + .withLocation(properties.get("location").orNull) + .withProperties(newProperties.asJava) + .withStorageTableIdentifier(TableIdentifier.parse(storageTableIdentifier)) + .create() + Some(new SparkView(catalog.name(), icebergView)) + } catch { + // TODO: Make sure the existing view is also a materialized view + case _: ViewAlreadyExistsException if allowExisting => None + } + } + } + +} diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala index 6dd1188b78e8..a5f5285131c7 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala @@ -18,6 +18,11 @@ */ package org.apache.spark.sql.execution.datasources.v2 +import org.apache.iceberg.catalog.Namespace +import org.apache.iceberg.catalog.TableIdentifier +import org.apache.iceberg.exceptions +import org.apache.iceberg.spark.SparkCatalog +import org.apache.iceberg.view.View import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.NoSuchViewException import org.apache.spark.sql.catalyst.expressions.Attribute @@ -30,6 +35,31 @@ case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boo override lazy val output: Seq[Attribute] = Nil override protected def run(): Seq[InternalRow] = { + // If the catalog is a SparkCatalog, check for materialized view storage table cleanup + catalog match { + case sparkCatalog: SparkCatalog => + val icebergCatalog = sparkCatalog.icebergCatalog() + val icebergViewCatalog = icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] + var view: Option[View] = None + try { + view = Some(icebergViewCatalog.loadView(TableIdentifier.of(Namespace.of(ident.namespace(): _*), ident.name()))) + } catch { + case _: exceptions.NoSuchViewException => + if (!ifExists) { + throw new NoSuchViewException(ident) + } + } + // if view is a materialized view, drop the storage table first + view.foreach { v => + val storageTable = v.currentVersion().storageTable() + if (storageTable != null) { + val storageIdent = Identifier.of(storageTable.namespace().levels(), storageTable.name()) + sparkCatalog.dropTable(storageIdent) + } + } + case _ => // not a SparkCatalog, skip MV cleanup + } + val dropped = catalog.dropView(ident) if (!dropped && !ifExists) { throw new NoSuchViewException(ident) diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala index da540f5891b7..3d34e3e9e7c1 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala @@ -141,6 +141,35 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi allowExisting, replace, _, + Some(materializedViewOptions), + _) => + CreateMaterializedViewExec( + catalog = viewCatalog, + ident = ident, + queryText = queryText, + columnAliases = columnAliases, + columnComments = columnComments, + queryColumnNames = queryColumnNames, + viewSchema = query.schema, + comment = comment, + properties = properties, + allowExisting = allowExisting, + replace = replace, + storageTableIdentifier = materializedViewOptions.storageTableIdentifier) :: Nil + + case CreateIcebergView( + ResolvedIdentifier(viewCatalog: ViewCatalog, ident), + queryText, + query, + columnAliases, + columnComments, + queryColumnNames, + comment, + properties, + allowExisting, + replace, + _, + None, _) => CreateV2ViewExec( catalog = viewCatalog, diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java new file mode 100644 index 000000000000..cb2edabd2cc2 --- /dev/null +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.extensions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Map; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.ParameterizedTestExtension; +import org.apache.iceberg.Parameters; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.RuntimeIOException; +import org.apache.iceberg.inmemory.InMemoryCatalog; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.spark.MaterializedViewUtil; +import org.apache.iceberg.spark.SparkCatalog; +import org.apache.iceberg.spark.SparkCatalogConfig; +import org.apache.iceberg.spark.source.SparkMaterializedView; +import org.apache.iceberg.spark.source.SparkView; +import org.apache.iceberg.view.RefreshState; +import org.apache.iceberg.view.RefreshStateParser; +import org.apache.iceberg.view.SourceTableState; +import org.apache.iceberg.view.View; +import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; +import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; +import org.apache.spark.sql.connector.catalog.CatalogPlugin; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.connector.catalog.ViewCatalog; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(ParameterizedTestExtension.class) +public class TestMaterializedViews extends ExtensionsTestBase { + private static final Namespace NAMESPACE = Namespace.of("default"); + private final String tableName = "table"; + private final String materializedViewName = "materialized_view"; + + @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") + protected static Object[][] parameters() { + Map properties = + Maps.newHashMap(SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.properties()); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, "file:" + getTempWarehouseDir()); + properties.put(CatalogProperties.CATALOG_IMPL, InMemoryCatalogWithLocalFileIO.class.getName()); + return new Object[][] { + { + SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.catalogName(), + SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.implementation(), + properties + } + }; + } + + private static String getTempWarehouseDir() { + try { + File tempDir = Files.createTempDirectory("warehouse-").toFile(); + tempDir.deleteOnExit(); + return tempDir.getAbsolutePath(); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @BeforeEach + @Override + public void before() { + // Set up a simple InMemoryCatalog as validation catalog to avoid base class + // configureValidationCatalog() failing on our custom catalog-impl. + this.validationCatalog = new InMemoryCatalog(); + this.validationNamespaceCatalog = + (org.apache.iceberg.catalog.SupportsNamespaces) validationCatalog; + + spark.conf().set("spark.sql.catalog." + catalogName, implementation); + catalogConfig.forEach( + (key, value) -> spark.conf().set("spark.sql.catalog." + catalogName + "." + key, value)); + + sql("CREATE NAMESPACE IF NOT EXISTS default"); + spark.conf().set("spark.sql.defaultCatalog", catalogName); + sql("USE %s", catalogName); + sql("CREATE NAMESPACE IF NOT EXISTS %s", NAMESPACE); + sql("CREATE TABLE %s (id INT, data STRING)", tableName); + } + + @AfterEach + public void removeTable() { + sql("USE %s", catalogName); + sql("DROP VIEW IF EXISTS %s", materializedViewName); + sql("DROP TABLE IF EXISTS %s", tableName); + } + + @TestTemplate + public void testStorageTableFieldOnViewVersion() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + View view = loadIcebergView(); + // storage-table should be set on the view version, not as a property + assertThat(view.currentVersion().storageTable()).isNotNull(); + assertThat(view.currentVersion().storageTable().name()) + .isEqualTo(materializedViewName + "__storage"); + assertThat(view.currentVersion().storageTable().namespace()) + .isEqualTo(NAMESPACE); + } + + @TestTemplate + public void testNeverRefreshedMvIsNotFresh() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // A newly created MV has no snapshots on its storage table, so it's not fresh. + // loadView should succeed (returns stale view) + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); + } catch (NoSuchViewException e) { + fail("Materialized view not found"); + } + } + + @TestTemplate + public void testReadFromStorageTableWhenFresh() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + simulateRefresh(); + + // Fresh MV: loadTable should return SparkMaterializedView + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Fresh materialized view should be loadable as a table"); + } + + // Fresh MV: loadView should throw since the engine should use loadTable instead + assertThatThrownBy(() -> sparkViewCatalog().loadView(viewIdentifier())) + .isInstanceOf(IllegalStateException.class); + } + + @TestTemplate + public void testFallbackToViewWhenStale() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + simulateRefresh(); + + // Insert more data to invalidate the refresh + sql("INSERT INTO %s VALUES (3, 'c')", tableName); + + // Stale MV: loadView should return SparkView (falls back to query execution) + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())) + .isInstanceOf(SparkView.class); + } catch (NoSuchViewException e) { + fail("Stale materialized view should be loadable as a view"); + } + + // Stale MV: loadTable should not resolve to the MV's storage table + assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(NoSuchTableException.class); + } + + @TestTemplate + public void testStorageTableCreatedBeforeMvMetadata() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // The storage table should exist + String storageTableName = + MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier( + Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName)) + .name(); + assertThat(sql("SHOW TABLES")) + .anySatisfy(row -> assertThat(row[1]).isEqualTo(storageTableName)); + } + + @TestTemplate + public void testDefaultStorageTableNaming() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // Default naming should be __storage + String expectedStorageTableName = materializedViewName + "__storage"; + assertThat(sql("SHOW TABLES")) + .anySatisfy(row -> assertThat(row[1]).isEqualTo(expectedStorageTableName)); + } + + @TestTemplate + public void testStoredAsClause() { + String customTableName = "custom_table_name"; + sql( + "CREATE MATERIALIZED VIEW %s STORED AS '%s' AS SELECT id, data FROM %s", + materializedViewName, customTableName, tableName); + + // Assert that the storage table with the custom name is in the list of tables + assertThat(sql("SHOW TABLES")).anySatisfy(row -> assertThat(row[1]).isEqualTo(customTableName)); + } + + private void simulateRefresh() { + View view = loadIcebergView(); + org.apache.iceberg.catalog.TableIdentifier storageTableId = + view.currentVersion().storageTable(); + + // Get the base table's current snapshot ID + long baseSnapshotId = + (Long) + sql( + "SELECT snapshot_id FROM %s.%s.%s.snapshots ORDER BY committed_at DESC LIMIT 1", + catalogName, NAMESPACE, tableName) + .get(0)[0]; + + // Build refresh state matching the current view version and source table state + RefreshState refreshState = + new RefreshState( + view.currentVersion().versionId(), + Arrays.asList( + new SourceTableState( + tableName, + Arrays.asList(NAMESPACE.levels()), + null, + "test-uuid", + baseSnapshotId, + null)), + System.currentTimeMillis()); + String refreshStateJson = RefreshStateParser.toJson(refreshState); + + // Write data to storage table with refresh-state in the snapshot summary + String storageTableRef = + String.format("%s.%s.%s", catalogName, NAMESPACE, storageTableId.name()); + try { + spark + .sql(String.format("SELECT id, data FROM %s.%s.%s", catalogName, NAMESPACE, tableName)) + .writeTo(storageTableRef) + .option( + "snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) + .append(); + } catch (NoSuchTableException e) { + throw new RuntimeException("Storage table not found during simulated refresh", e); + } + } + + private ViewCatalog sparkViewCatalog() { + CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); + return (ViewCatalog) catalogPlugin; + } + + private TableCatalog sparkTableCatalog() { + CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); + return (TableCatalog) catalogPlugin; + } + + private Identifier viewIdentifier() { + return Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName); + } + + private SparkCatalog sparkCatalog() { + return (SparkCatalog) spark.sessionState().catalogManager().catalog(catalogName); + } + + private View loadIcebergView() { + org.apache.iceberg.catalog.ViewCatalog icebergViewCatalog = + (org.apache.iceberg.catalog.ViewCatalog) sparkCatalog().icebergCatalog(); + return icebergViewCatalog.loadView( + TableIdentifier.of(NAMESPACE, materializedViewName)); + } + + // Required to be public since it is loaded by org.apache.iceberg.CatalogUtil.loadCatalog + public static class InMemoryCatalogWithLocalFileIO extends InMemoryCatalog { + private FileIO localFileIO; + + @Override + public void initialize(String name, Map properties) { + super.initialize(name, properties); + localFileIO = new LocalFileIO(); + } + + @Override + protected TableOperations newTableOps(TableIdentifier tableIdentifier) { + return new InMemoryTableOperations(localFileIO, tableIdentifier); + } + + @Override + protected InMemoryCatalog.InMemoryViewOperations newViewOps(TableIdentifier identifier) { + return new InMemoryViewOperations(localFileIO, identifier); + } + } + + private static class LocalFileIO implements FileIO { + + private static String stripFilePrefix(String path) { + return path.startsWith("file:") ? path.substring(5) : path; + } + + @Override + public InputFile newInputFile(String path) { + return org.apache.iceberg.Files.localInput(stripFilePrefix(path)); + } + + @Override + public OutputFile newOutputFile(String path) { + String stripped = stripFilePrefix(path); + java.io.File parent = new java.io.File(stripped).getParentFile(); + if (!parent.isDirectory()) { + parent.mkdirs(); + } + return org.apache.iceberg.Files.localOutput(stripped); + } + + @Override + public void deleteFile(String path) { + if (!new File(stripFilePrefix(path)).delete()) { + throw new RuntimeIOException("Failed to delete file: " + path); + } + } + } +} diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java new file mode 100644 index 000000000000..a30c5f671176 --- /dev/null +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark; + +import org.apache.spark.sql.connector.catalog.Identifier; + +public class MaterializedViewUtil { + + private MaterializedViewUtil() {} + + private static final String MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX = "__storage"; + + public static Identifier getDefaultMaterializedViewStorageTableIdentifier( + Identifier viewIdentifier) { + return Identifier.of( + viewIdentifier.namespace(), + viewIdentifier.name() + MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX); + } +} diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index 40db152076c8..b1c73a2faf87 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -58,6 +58,7 @@ import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.spark.actions.SparkActions; import org.apache.iceberg.spark.source.SparkChangelogTable; +import org.apache.iceberg.spark.source.SparkMaterializedView; import org.apache.iceberg.spark.source.SparkTable; import org.apache.iceberg.spark.source.SparkView; import org.apache.iceberg.spark.source.StagedSparkTable; @@ -598,7 +599,12 @@ public View loadView(Identifier ident) throws NoSuchViewException { if (null != asViewCatalog) { try { org.apache.iceberg.view.View view = asViewCatalog.loadView(buildIdentifier(ident)); - return new SparkView(catalogName, view); + if (isMaterializedView(view) && isFresh(view)) { + throw new IllegalStateException( + "Materialized view is fresh. loadTable should be attempted instead."); + } else { + return new SparkView(catalogName, view); + } } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { throw new NoSuchViewException(ident); } @@ -607,6 +613,81 @@ public View loadView(Identifier ident) throws NoSuchViewException { throw new NoSuchViewException(ident); } + private boolean isMaterializedView(org.apache.iceberg.view.View view) { + return view.currentVersion().storageTable() != null; + } + + private org.apache.iceberg.catalog.TableIdentifier getStorageTableId( + org.apache.iceberg.view.View view) { + org.apache.iceberg.catalog.TableIdentifier storageTable = + view.currentVersion().storageTable(); + Preconditions.checkState( + storageTable != null, "Storage table identifier is not set for materialized view."); + return storageTable; + } + + private Table loadStorageTable(org.apache.iceberg.view.View view) { + org.apache.iceberg.catalog.TableIdentifier storageTableId = getStorageTableId(view); + try { + Identifier sparkIdent = + Identifier.of(storageTableId.namespace().levels(), storageTableId.name()); + return loadTable(sparkIdent); + } catch (NoSuchTableException e) { + throw new IllegalStateException("Unable to load storage table for materialized view.", e); + } + } + + private boolean isFresh(org.apache.iceberg.view.View view) { + Table sparkStorageTable = loadStorageTable(view); + org.apache.iceberg.Table storageTable = ((SparkTable) sparkStorageTable).table(); + if (storageTable.currentSnapshot() == null) { + return false; + } + + String refreshStateJson = + storageTable + .currentSnapshot() + .summary() + .get(org.apache.iceberg.view.RefreshState.REFRESH_STATE_SUMMARY_KEY); + if (refreshStateJson == null) { + return false; + } + + org.apache.iceberg.view.RefreshState refreshState = + org.apache.iceberg.view.RefreshStateParser.fromJson(refreshStateJson); + + if (refreshState.viewVersionId() != view.currentVersion().versionId()) { + return false; + } + + for (org.apache.iceberg.view.SourceState sourceState : refreshState.sourceStates()) { + if (sourceState instanceof org.apache.iceberg.view.SourceTableState) { + org.apache.iceberg.view.SourceTableState tableState = + (org.apache.iceberg.view.SourceTableState) sourceState; + org.apache.iceberg.catalog.TableIdentifier sourceId = + org.apache.iceberg.catalog.TableIdentifier.of( + org.apache.iceberg.catalog.Namespace.of( + tableState.namespace().toArray(new String[0])), + tableState.name()); + try { + org.apache.iceberg.Table sourceTable = + ((org.apache.iceberg.catalog.Catalog) icebergCatalog()).loadTable(sourceId); + long currentSnapshotId = + sourceTable.currentSnapshot() == null + ? -1 + : sourceTable.currentSnapshot().snapshotId(); + if (currentSnapshotId != tableState.snapshotId()) { + return false; + } + } catch (Exception e) { + return false; + } + } + } + + return true; + } + @Override public View createView(ViewInfo viewInfo) throws ViewAlreadyExistsException, NoSuchNamespaceException { @@ -897,6 +978,19 @@ private Table load(Identifier ident, TimeTravel timeTravel) throws NoSuchTableEx return loadPath((PathIdentifier) ident, timeTravel); } + // Check if materialized view. If fresh, return the SparkMaterializedView. + if (null != asViewCatalog) { + try { + org.apache.iceberg.view.View view = asViewCatalog.loadView(buildIdentifier(ident)); + if (isMaterializedView(view) && isFresh(view)) { + Table storageTable = loadStorageTable(view); + return new SparkMaterializedView(catalogName, view, storageTable); + } + } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { + // Ignore. Just process as a normal table. + } + } + try { org.apache.iceberg.Table table = icebergCatalog.loadTable(buildIdentifier(ident)); return SparkTable.create(table, timeTravel); diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java new file mode 100644 index 000000000000..0c01ae449292 --- /dev/null +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.source; + +import java.util.Set; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.view.View; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.connector.catalog.SupportsRead; +import org.apache.spark.sql.connector.catalog.Table; +import org.apache.spark.sql.connector.catalog.TableCapability; +import org.apache.spark.sql.connector.read.ScanBuilder; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +public class SparkMaterializedView extends SparkView implements SupportsRead { + private final Table storageTable; + private SparkSession lazySpark; + + public SparkMaterializedView(String catalogName, View icebergView, Table storageTable) { + super(catalogName, icebergView); + this.storageTable = storageTable; + } + + @Override + public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) { + return ((SupportsRead) storageTable).newScanBuilder(options); + } + + private SparkSession sparkSession() { + if (lazySpark == null) { + this.lazySpark = SparkSession.active(); + } + + return lazySpark; + } + + @Override + public Set capabilities() { + return ImmutableSet.of(TableCapability.BATCH_READ); + } +} diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java index b20c87619ed8..d02353a85b9b 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java @@ -81,7 +81,17 @@ public enum SparkCatalogConfig { ImmutableMap.of( "type", "hive", "default-namespace", "default", - "unique-table-location", "true")); + "unique-table-location", "true")), + SPARK_WITH_MATERIALIZED_VIEWS( + "spark_with_mvs", + SparkCatalog.class.getName(), + ImmutableMap.of( + CatalogProperties.CATALOG_IMPL, + InMemoryCatalog.class.getName(), + "default-namespace", + "default", + "cache-enabled", + "false")); private final String catalogName; private final String implementation; From e6a5abe075337c23a9a87504da2b087df782e45b Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 24 Mar 2026 14:18:20 -0700 Subject: [PATCH 07/22] Fix scalestyle violations in MV Spark extensions - Replace block imports with individual imports - Remove empty line separating import groups - Break long lines to stay within 120 char limit - Applied to both v3.5 and v4.1 versions --- .../v2/CreateMaterializedViewExec.scala | 21 +++++++++++++------ .../datasources/v2/DropV2ViewExec.scala | 4 +++- .../v2/CreateMaterializedViewExec.scala | 21 +++++++++++++------ .../datasources/v2/DropV2ViewExec.scala | 4 +++- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala index 79cc5eb02548..b7587a29134a 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -19,11 +19,16 @@ package org.apache.spark.sql.execution.datasources.v2 +import scala.collection.JavaConverters._ -import org.apache.iceberg.catalog.{Namespace, TableIdentifier} +import org.apache.iceberg.catalog.Namespace +import org.apache.iceberg.catalog.TableIdentifier import org.apache.iceberg.relocated.com.google.common.base.Preconditions import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap -import org.apache.iceberg.spark.{MaterializedViewUtil, Spark3Util, SparkCatalog, SparkSchemaUtil} +import org.apache.iceberg.spark.MaterializedViewUtil +import org.apache.iceberg.spark.Spark3Util +import org.apache.iceberg.spark.SparkCatalog +import org.apache.iceberg.spark.SparkSchemaUtil import org.apache.iceberg.spark.source.SparkView import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException @@ -34,8 +39,6 @@ import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.types.StructType -import scala.collection.JavaConverters._ - case class CreateMaterializedViewExec( catalog: ViewCatalog, ident: Identifier, @@ -123,7 +126,10 @@ case class CreateMaterializedViewExec( catalog.dropView(ident) } // FIXME: replaceView API doesn't exist in Spark 3.5 - val icebergView = catalog.asInstanceOf[SparkCatalog].icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( + val viewCatalog = catalog.asInstanceOf[SparkCatalog] + .icebergCatalog() + .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] + val icebergView = viewCatalog.buildView( Spark3Util.identifierToTableIdentifier(ident)) .withDefaultCatalog(currentCatalog) .withDefaultNamespace(Namespace.of(currentNamespace: _*)) @@ -138,7 +144,10 @@ case class CreateMaterializedViewExec( } else { try { // CREATE VIEW [IF NOT EXISTS] - val icebergView = catalog.asInstanceOf[SparkCatalog].icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( + val viewCatalog = catalog.asInstanceOf[SparkCatalog] + .icebergCatalog() + .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] + val icebergView = viewCatalog.buildView( Spark3Util.identifierToTableIdentifier(ident)) .withDefaultCatalog(currentCatalog) .withDefaultNamespace(Namespace.of(currentNamespace: _*)) diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala index a5f5285131c7..84c111ae82ef 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala @@ -42,7 +42,9 @@ case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boo val icebergViewCatalog = icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] var view: Option[View] = None try { - view = Some(icebergViewCatalog.loadView(TableIdentifier.of(Namespace.of(ident.namespace(): _*), ident.name()))) + val ns = Namespace.of(ident.namespace(): _*) + val viewId = TableIdentifier.of(ns, ident.name()) + view = Some(icebergViewCatalog.loadView(viewId)) } catch { case _: exceptions.NoSuchViewException => if (!ifExists) { diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala index 3e27ff78acbb..781aade9fc2e 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -19,11 +19,16 @@ package org.apache.spark.sql.execution.datasources.v2 +import scala.jdk.CollectionConverters._ -import org.apache.iceberg.catalog.{Namespace, TableIdentifier} +import org.apache.iceberg.catalog.Namespace +import org.apache.iceberg.catalog.TableIdentifier import org.apache.iceberg.relocated.com.google.common.base.Preconditions import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap -import org.apache.iceberg.spark.{MaterializedViewUtil, Spark3Util, SparkCatalog, SparkSchemaUtil} +import org.apache.iceberg.spark.MaterializedViewUtil +import org.apache.iceberg.spark.Spark3Util +import org.apache.iceberg.spark.SparkCatalog +import org.apache.iceberg.spark.SparkSchemaUtil import org.apache.iceberg.spark.source.SparkView import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException @@ -34,8 +39,6 @@ import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.types.StructType -import scala.jdk.CollectionConverters._ - case class CreateMaterializedViewExec( catalog: ViewCatalog, ident: Identifier, @@ -123,7 +126,10 @@ case class CreateMaterializedViewExec( catalog.dropView(ident) } // FIXME: replaceView API doesn't exist in Spark 3.5 - val icebergView = catalog.asInstanceOf[SparkCatalog].icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( + val viewCatalog = catalog.asInstanceOf[SparkCatalog] + .icebergCatalog() + .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] + val icebergView = viewCatalog.buildView( Spark3Util.identifierToTableIdentifier(ident)) .withDefaultCatalog(currentCatalog) .withDefaultNamespace(Namespace.of(currentNamespace: _*)) @@ -138,7 +144,10 @@ case class CreateMaterializedViewExec( } else { try { // CREATE VIEW [IF NOT EXISTS] - val icebergView = catalog.asInstanceOf[SparkCatalog].icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog].buildView( + val viewCatalog = catalog.asInstanceOf[SparkCatalog] + .icebergCatalog() + .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] + val icebergView = viewCatalog.buildView( Spark3Util.identifierToTableIdentifier(ident)) .withDefaultCatalog(currentCatalog) .withDefaultNamespace(Namespace.of(currentNamespace: _*)) diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala index a5f5285131c7..84c111ae82ef 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala @@ -42,7 +42,9 @@ case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boo val icebergViewCatalog = icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] var view: Option[View] = None try { - view = Some(icebergViewCatalog.loadView(TableIdentifier.of(Namespace.of(ident.namespace(): _*), ident.name()))) + val ns = Namespace.of(ident.namespace(): _*) + val viewId = TableIdentifier.of(ns, ident.name()) + view = Some(icebergViewCatalog.loadView(viewId)) } catch { case _: exceptions.NoSuchViewException => if (!ifExists) { From 7033883cf3c582b832c584635004f3e15869d2eb Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 24 Mar 2026 14:34:00 -0700 Subject: [PATCH 08/22] Fix spotless formatting in ViewBuilder.java --- api/src/main/java/org/apache/iceberg/view/ViewBuilder.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java b/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java index 5e025f19bb3a..7fac2e48d55f 100644 --- a/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java +++ b/api/src/main/java/org/apache/iceberg/view/ViewBuilder.java @@ -63,8 +63,7 @@ default ViewBuilder withLocation(String location) { * @return this for method chaining */ default ViewBuilder withStorageTableIdentifier(TableIdentifier storageTableIdentifier) { - throw new UnsupportedOperationException( - "Setting a storage table identifier is not supported"); + throw new UnsupportedOperationException("Setting a storage table identifier is not supported"); } /** From 70021d4f6649831424fa5dc2012f34ed9f29a729 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 24 Mar 2026 14:38:56 -0700 Subject: [PATCH 09/22] Fix remaining spotless issues --- .../iceberg/view/RefreshStateParser.java | 6 +- .../apache/iceberg/view/SourceViewState.java | 6 +- .../iceberg/view/TestRefreshStateParser.java | 21 ++----- .../analysis/RewriteViewCommands.scala | 6 +- .../IcebergSparkSqlExtensionsParser.scala | 11 ++-- .../v2/CreateMaterializedViewExec.scala | 61 ++++++++++--------- .../extensions/TestMaterializedViews.java | 12 ++-- .../apache/iceberg/spark/SparkCatalog.java | 3 +- 8 files changed, 56 insertions(+), 70 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/view/RefreshStateParser.java b/core/src/main/java/org/apache/iceberg/view/RefreshStateParser.java index f032f93604db..019f7c1d32ad 100644 --- a/core/src/main/java/org/apache/iceberg/view/RefreshStateParser.java +++ b/core/src/main/java/org/apache/iceberg/view/RefreshStateParser.java @@ -58,8 +58,7 @@ public static void toJson(RefreshState refreshState, JsonGenerator generator) th generator.writeStartObject(); generator.writeNumberField(VIEW_VERSION_ID, refreshState.viewVersionId()); - generator.writeNumberField( - REFRESH_START_TIMESTAMP_MS, refreshState.refreshStartTimestampMs()); + generator.writeNumberField(REFRESH_START_TIMESTAMP_MS, refreshState.refreshStartTimestampMs()); generator.writeArrayFieldStart(SOURCE_STATES); for (SourceState sourceState : refreshState.sourceStates()) { @@ -124,8 +123,7 @@ public static RefreshState fromJson(JsonNode node) { private static SourceState parseSourceState(JsonNode node) { String type = JsonUtil.getString(TYPE, node); String name = JsonUtil.getString(NAME, node); - List namespace = - Arrays.asList(JsonUtil.getStringArray(JsonUtil.get(NAMESPACE, node))); + List namespace = Arrays.asList(JsonUtil.getStringArray(JsonUtil.get(NAMESPACE, node))); String catalog = JsonUtil.getStringOrNull(CATALOG, node); String uuid = JsonUtil.getString(UUID, node); diff --git a/core/src/main/java/org/apache/iceberg/view/SourceViewState.java b/core/src/main/java/org/apache/iceberg/view/SourceViewState.java index 266dd171f0c9..87732806b045 100644 --- a/core/src/main/java/org/apache/iceberg/view/SourceViewState.java +++ b/core/src/main/java/org/apache/iceberg/view/SourceViewState.java @@ -33,11 +33,7 @@ public class SourceViewState implements SourceState { private final int versionId; public SourceViewState( - String name, - List namespace, - @Nullable String catalog, - String uuid, - int versionId) { + String name, List namespace, @Nullable String catalog, String uuid, int versionId) { Preconditions.checkArgument(name != null, "Source view name is required"); Preconditions.checkArgument( namespace != null && !namespace.isEmpty(), "Source view namespace is required"); diff --git a/core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java b/core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java index cc0ab06ca1cd..a7fbd8dd44a8 100644 --- a/core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java +++ b/core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java @@ -91,21 +91,10 @@ public void testRoundTripSourceViewState() { @Test public void testRoundTripMixedSourceStates() { SourceTableState tableState = - new SourceTableState( - "events", - Arrays.asList("default"), - null, - "uuid-1", - 100L, - "main"); + new SourceTableState("events", Arrays.asList("default"), null, "uuid-1", 100L, "main"); SourceViewState viewState = - new SourceViewState( - "event_summary", - Arrays.asList("default"), - null, - "uuid-2", - 3); + new SourceViewState("event_summary", Arrays.asList("default"), null, "uuid-2", 3); RefreshState refreshState = new RefreshState(1, Arrays.asList(tableState, viewState), 1573518435000L); @@ -145,15 +134,13 @@ public void testParseSpecExample() { SourceTableState tableState = (SourceTableState) parsed.sourceStates().get(0); Assertions.assertThat(tableState.name()).isEqualTo("events"); Assertions.assertThat(tableState.namespace()).containsExactly("default"); - Assertions.assertThat(tableState.uuid()) - .isEqualTo("d4a10b5c-1e8a-4b72-9d67-3f4a8c9e1b2d"); + Assertions.assertThat(tableState.uuid()).isEqualTo("d4a10b5c-1e8a-4b72-9d67-3f4a8c9e1b2d"); Assertions.assertThat(tableState.snapshotId()).isEqualTo(6148331192489823102L); } @Test public void testEmptySourceStates() { - RefreshState refreshState = - new RefreshState(1, Collections.emptyList(), 1573518435000L); + RefreshState refreshState = new RefreshState(1, Collections.emptyList(), 1573518435000L); String json = RefreshStateParser.toJson(refreshState); RefreshState parsed = RefreshStateParser.fromJson(json); diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala index c6b988255cc6..3ffef317af42 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala @@ -40,7 +40,11 @@ import scala.collection.mutable * ResolveSessionCatalog exits early for some v2 View commands, * thus they are pre-substituted here and then handled in ResolveViews */ -case class RewriteViewCommands(spark: SparkSession, materializedViewOptions: Option[MaterializedViewOptions] = None) extends Rule[LogicalPlan] with LookupCatalog { +case class RewriteViewCommands( + spark: SparkSession, + materializedViewOptions: Option[MaterializedViewOptions] = None) + extends Rule[LogicalPlan] + with LookupCatalog { import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala index e74eceddfc05..a0070a9ba3fc 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala @@ -54,7 +54,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) private lazy val substitutor = substitutorCtor.newInstance(SQLConf.get) private lazy val astBuilder = new IcebergSqlExtensionsAstBuilder(delegate) - private lazy final val CREATE_MATERIALIZED_VIEW_PATTERN = "(?i)(CREATE)\\s+MATERIALIZED\\s+(VIEW)".r + private lazy final val CREATE_MATERIALIZED_VIEW_PATTERN = + "(?i)(CREATE)\\s+MATERIALIZED\\s+(VIEW)".r private lazy final val MATERIALIZED_VIEW_STORED_AS_PATTERN = "(?i)STORED AS\\s*'(\\w+)'\\s*".r /** @@ -146,9 +147,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) } .asInstanceOf[LogicalPlan] } else if (isCreateMaterializedView(sqlText)) { - RewriteViewCommands(SparkSession.active, Option(getMaterializedViewOptions(sqlText))).apply( - delegate.parsePlan(getCreateMaterializedViewStatement(sqlText)) - ) + RewriteViewCommands(SparkSession.active, Option(getMaterializedViewOptions(sqlText))) + .apply(delegate.parsePlan(getCreateMaterializedViewStatement(sqlText))) } else { RewriteViewCommands(SparkSession.active).apply(delegateParse(sqlText)) } @@ -159,7 +159,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) } private def getCreateMaterializedViewStatement(sqlText: String): String = { - val createViewSql = CREATE_MATERIALIZED_VIEW_PATTERN.replaceAllIn(sqlText, m => m.group(1) + " " + m.group(2)) + val createViewSql = + CREATE_MATERIALIZED_VIEW_PATTERN.replaceAllIn(sqlText, m => m.group(1) + " " + m.group(2)) MATERIALIZED_VIEW_STORED_AS_PATTERN.replaceAllIn(createViewSql, "") } diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala index 781aade9fc2e..a055794fa284 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.spark.sql.execution.datasources.v2 import scala.jdk.CollectionConverters._ @@ -40,18 +39,19 @@ import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.types.StructType case class CreateMaterializedViewExec( - catalog: ViewCatalog, - ident: Identifier, - queryText: String, - viewSchema: StructType, - columnAliases: Seq[String], - columnComments: Seq[Option[String]], - queryColumnNames: Seq[String], - comment: Option[String], - properties: Map[String, String], - allowExisting: Boolean, - replace: Boolean, - storageTableIdentifier: Option[String]) extends LeafV2CommandExec { + catalog: ViewCatalog, + ident: Identifier, + queryText: String, + viewSchema: StructType, + columnAliases: Seq[String], + columnComments: Seq[Option[String]], + queryColumnNames: Seq[String], + comment: Option[String], + properties: Map[String, String], + allowExisting: Boolean, + replace: Boolean, + storageTableIdentifier: Option[String]) + extends LeafV2CommandExec { override def output: Seq[Attribute] = Nil @@ -66,8 +66,7 @@ case class CreateMaterializedViewExec( storageTableCatalogName.equals(catalog.name()), "Storage table identifier must be in the same catalog as the view." + " Found storage table in catalog: %s, expected: %s.", - Array[Object](storageTableCatalogName, catalog.name()) - ) + Array[Object](storageTableCatalogName, catalog.name())) catalogAndIdentifier.identifier() } case None => MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier(ident) @@ -77,10 +76,13 @@ case class CreateMaterializedViewExec( // Per spec: "The storage table must exist and be accessible before the // materialized view metadata is committed." // A newly created MV has a storage table with no snapshots until a refresh is performed. - catalog.asInstanceOf[SparkCatalog].createTable( - sparkStorageTableIdentifier, - viewSchema, new Array[Transform](0), ImmutableMap.of[String, String]() - ) + catalog + .asInstanceOf[SparkCatalog] + .createTable( + sparkStorageTableIdentifier, + viewSchema, + new Array[Transform](0), + ImmutableMap.of[String, String]()) // Step 2: Create the MV view metadata with a storage-table reference try { @@ -109,28 +111,30 @@ case class CreateMaterializedViewExec( private def createView(storageTableIdentifier: String): Option[View] = { val icebergSchema = SparkSchemaUtil.convert(viewSchema) val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name - val currentCatalog = if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null + val currentCatalog = + if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null val currentNamespace = session.sessionState.catalogManager.currentNamespace val engineVersion = "Spark " + org.apache.spark.SPARK_VERSION val newProperties = properties ++ comment.map(ViewCatalog.PROP_COMMENT -> _) + - (ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion, + ( + ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion, ViewCatalog.PROP_ENGINE_VERSION -> engineVersion) + ("queryColumnNames" -> queryColumnNames.mkString(",")) - if (replace) { // CREATE OR REPLACE VIEW if (catalog.viewExists(ident)) { catalog.dropView(ident) } // FIXME: replaceView API doesn't exist in Spark 3.5 - val viewCatalog = catalog.asInstanceOf[SparkCatalog] + val viewCatalog = catalog + .asInstanceOf[SparkCatalog] .icebergCatalog() .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val icebergView = viewCatalog.buildView( - Spark3Util.identifierToTableIdentifier(ident)) + val icebergView = viewCatalog + .buildView(Spark3Util.identifierToTableIdentifier(ident)) .withDefaultCatalog(currentCatalog) .withDefaultNamespace(Namespace.of(currentNamespace: _*)) .withQuery("spark", queryText) @@ -144,11 +148,12 @@ case class CreateMaterializedViewExec( } else { try { // CREATE VIEW [IF NOT EXISTS] - val viewCatalog = catalog.asInstanceOf[SparkCatalog] + val viewCatalog = catalog + .asInstanceOf[SparkCatalog] .icebergCatalog() .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val icebergView = viewCatalog.buildView( - Spark3Util.identifierToTableIdentifier(ident)) + val icebergView = viewCatalog + .buildView(Spark3Util.identifierToTableIdentifier(ident)) .withDefaultCatalog(currentCatalog) .withDefaultNamespace(Namespace.of(currentNamespace: _*)) .withQuery("spark", queryText) diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index cb2edabd2cc2..66c705c2c541 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -127,8 +127,7 @@ public void testStorageTableFieldOnViewVersion() { assertThat(view.currentVersion().storageTable()).isNotNull(); assertThat(view.currentVersion().storageTable().name()) .isEqualTo(materializedViewName + "__storage"); - assertThat(view.currentVersion().storageTable().namespace()) - .isEqualTo(NAMESPACE); + assertThat(view.currentVersion().storageTable().namespace()).isEqualTo(NAMESPACE); } @TestTemplate @@ -176,8 +175,7 @@ public void testFallbackToViewWhenStale() { // Stale MV: loadView should return SparkView (falls back to query execution) try { - assertThat(sparkViewCatalog().loadView(viewIdentifier())) - .isInstanceOf(SparkView.class); + assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); } catch (NoSuchViewException e) { fail("Stale materialized view should be loadable as a view"); } @@ -256,8 +254,7 @@ private void simulateRefresh() { spark .sql(String.format("SELECT id, data FROM %s.%s.%s", catalogName, NAMESPACE, tableName)) .writeTo(storageTableRef) - .option( - "snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) + .option("snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) .append(); } catch (NoSuchTableException e) { throw new RuntimeException("Storage table not found during simulated refresh", e); @@ -285,8 +282,7 @@ private SparkCatalog sparkCatalog() { private View loadIcebergView() { org.apache.iceberg.catalog.ViewCatalog icebergViewCatalog = (org.apache.iceberg.catalog.ViewCatalog) sparkCatalog().icebergCatalog(); - return icebergViewCatalog.loadView( - TableIdentifier.of(NAMESPACE, materializedViewName)); + return icebergViewCatalog.loadView(TableIdentifier.of(NAMESPACE, materializedViewName)); } // Required to be public since it is loaded by org.apache.iceberg.CatalogUtil.loadCatalog diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index b1c73a2faf87..fc9863363fa4 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -619,8 +619,7 @@ private boolean isMaterializedView(org.apache.iceberg.view.View view) { private org.apache.iceberg.catalog.TableIdentifier getStorageTableId( org.apache.iceberg.view.View view) { - org.apache.iceberg.catalog.TableIdentifier storageTable = - view.currentVersion().storageTable(); + org.apache.iceberg.catalog.TableIdentifier storageTable = view.currentVersion().storageTable(); Preconditions.checkState( storageTable != null, "Storage table identifier is not set for materialized view."); return storageTable; From 5a97c32213e73c30ffbf637b64787d7e43c480c1 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 24 Mar 2026 17:07:01 -0700 Subject: [PATCH 10/22] Fix spotless and scalestyle violations across v3.5 and v4.1 --- .../v2/CreateMaterializedViewExec.scala | 64 ++++++++++--------- .../extensions/TestMaterializedViews.java | 12 ++-- .../apache/iceberg/spark/SparkCatalog.java | 7 +- .../v2/CreateMaterializedViewExec.scala | 3 +- 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala index b7587a29134a..dd6fdcc5f212 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -16,11 +16,8 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.spark.sql.execution.datasources.v2 -import scala.collection.JavaConverters._ - import org.apache.iceberg.catalog.Namespace import org.apache.iceberg.catalog.TableIdentifier import org.apache.iceberg.relocated.com.google.common.base.Preconditions @@ -38,20 +35,22 @@ import org.apache.spark.sql.connector.catalog.View import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.types.StructType +import scala.collection.JavaConverters._ case class CreateMaterializedViewExec( - catalog: ViewCatalog, - ident: Identifier, - queryText: String, - viewSchema: StructType, - columnAliases: Seq[String], - columnComments: Seq[Option[String]], - queryColumnNames: Seq[String], - comment: Option[String], - properties: Map[String, String], - allowExisting: Boolean, - replace: Boolean, - storageTableIdentifier: Option[String]) extends LeafV2CommandExec { + catalog: ViewCatalog, + ident: Identifier, + queryText: String, + viewSchema: StructType, + columnAliases: Seq[String], + columnComments: Seq[Option[String]], + queryColumnNames: Seq[String], + comment: Option[String], + properties: Map[String, String], + allowExisting: Boolean, + replace: Boolean, + storageTableIdentifier: Option[String]) + extends LeafV2CommandExec { override def output: Seq[Attribute] = Nil @@ -66,8 +65,7 @@ case class CreateMaterializedViewExec( storageTableCatalogName.equals(catalog.name()), "Storage table identifier must be in the same catalog as the view." + " Found storage table in catalog: %s, expected: %s.", - Array[Object](storageTableCatalogName, catalog.name()) - ) + Array[Object](storageTableCatalogName, catalog.name())) catalogAndIdentifier.identifier() } case None => MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier(ident) @@ -77,10 +75,13 @@ case class CreateMaterializedViewExec( // Per spec: "The storage table must exist and be accessible before the // materialized view metadata is committed." // A newly created MV has a storage table with no snapshots until a refresh is performed. - catalog.asInstanceOf[SparkCatalog].createTable( - sparkStorageTableIdentifier, - viewSchema, new Array[Transform](0), ImmutableMap.of[String, String]() - ) + catalog + .asInstanceOf[SparkCatalog] + .createTable( + sparkStorageTableIdentifier, + viewSchema, + new Array[Transform](0), + ImmutableMap.of[String, String]()) // Step 2: Create the MV view metadata with a storage-table reference try { @@ -109,28 +110,30 @@ case class CreateMaterializedViewExec( private def createView(storageTableIdentifier: String): Option[View] = { val icebergSchema = SparkSchemaUtil.convert(viewSchema) val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name - val currentCatalog = if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null + val currentCatalog = + if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null val currentNamespace = session.sessionState.catalogManager.currentNamespace val engineVersion = "Spark " + org.apache.spark.SPARK_VERSION val newProperties = properties ++ comment.map(ViewCatalog.PROP_COMMENT -> _) + - (ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion, + ( + ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion, ViewCatalog.PROP_ENGINE_VERSION -> engineVersion) + ("queryColumnNames" -> queryColumnNames.mkString(",")) - if (replace) { // CREATE OR REPLACE VIEW if (catalog.viewExists(ident)) { catalog.dropView(ident) } // FIXME: replaceView API doesn't exist in Spark 3.5 - val viewCatalog = catalog.asInstanceOf[SparkCatalog] + val viewCatalog = catalog + .asInstanceOf[SparkCatalog] .icebergCatalog() .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val icebergView = viewCatalog.buildView( - Spark3Util.identifierToTableIdentifier(ident)) + val icebergView = viewCatalog + .buildView(Spark3Util.identifierToTableIdentifier(ident)) .withDefaultCatalog(currentCatalog) .withDefaultNamespace(Namespace.of(currentNamespace: _*)) .withQuery("spark", queryText) @@ -144,11 +147,12 @@ case class CreateMaterializedViewExec( } else { try { // CREATE VIEW [IF NOT EXISTS] - val viewCatalog = catalog.asInstanceOf[SparkCatalog] + val viewCatalog = catalog + .asInstanceOf[SparkCatalog] .icebergCatalog() .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val icebergView = viewCatalog.buildView( - Spark3Util.identifierToTableIdentifier(ident)) + val icebergView = viewCatalog + .buildView(Spark3Util.identifierToTableIdentifier(ident)) .withDefaultCatalog(currentCatalog) .withDefaultNamespace(Namespace.of(currentNamespace: _*)) .withQuery("spark", queryText) diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index a05eddfaece3..3a0aab5f266d 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -117,8 +117,7 @@ public void testStorageTableFieldOnViewVersion() { assertThat(view.currentVersion().storageTable()).isNotNull(); assertThat(view.currentVersion().storageTable().name()) .isEqualTo(materializedViewName + "__storage"); - assertThat(view.currentVersion().storageTable().namespace()) - .isEqualTo(NAMESPACE); + assertThat(view.currentVersion().storageTable().namespace()).isEqualTo(NAMESPACE); } @Test @@ -166,8 +165,7 @@ public void testFallbackToViewWhenStale() { // Stale MV: loadView should return SparkView (falls back to query execution) try { - assertThat(sparkViewCatalog().loadView(viewIdentifier())) - .isInstanceOf(SparkView.class); + assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); } catch (NoSuchViewException e) { fail("Stale materialized view should be loadable as a view"); } @@ -246,8 +244,7 @@ private void simulateRefresh() { spark .sql(String.format("SELECT id, data FROM %s.%s.%s", catalogName, NAMESPACE, tableName)) .writeTo(storageTableRef) - .option( - "snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) + .option("snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) .append(); } catch (NoSuchTableException e) { throw new RuntimeException("Storage table not found during simulated refresh", e); @@ -275,8 +272,7 @@ private SparkCatalog sparkCatalog() { private View loadIcebergView() { org.apache.iceberg.catalog.ViewCatalog icebergViewCatalog = (org.apache.iceberg.catalog.ViewCatalog) sparkCatalog().icebergCatalog(); - return icebergViewCatalog.loadView( - TableIdentifier.of(NAMESPACE, materializedViewName)); + return icebergViewCatalog.loadView(TableIdentifier.of(NAMESPACE, materializedViewName)); } // Required to be public since it is loaded by org.apache.iceberg.CatalogUtil.loadCatalog diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index 6dcdd074ffb6..118e6bbb7420 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -616,8 +616,7 @@ private boolean isMaterializedView(org.apache.iceberg.view.View view) { private org.apache.iceberg.catalog.TableIdentifier getStorageTableId( org.apache.iceberg.view.View view) { - org.apache.iceberg.catalog.TableIdentifier storageTable = - view.currentVersion().storageTable(); + org.apache.iceberg.catalog.TableIdentifier storageTable = view.currentVersion().storageTable(); Preconditions.checkState( storageTable != null, "Storage table identifier is not set for materialized view."); return storageTable; @@ -672,7 +671,9 @@ private boolean isFresh(org.apache.iceberg.view.View view) { org.apache.iceberg.Table sourceTable = ((org.apache.iceberg.catalog.Catalog) icebergCatalog()).loadTable(sourceId); long currentSnapshotId = - sourceTable.currentSnapshot() == null ? -1 : sourceTable.currentSnapshot().snapshotId(); + sourceTable.currentSnapshot() == null + ? -1 + : sourceTable.currentSnapshot().snapshotId(); if (currentSnapshotId != tableState.snapshotId()) { return false; } diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala index a055794fa284..c69417032709 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -18,8 +18,6 @@ */ package org.apache.spark.sql.execution.datasources.v2 -import scala.jdk.CollectionConverters._ - import org.apache.iceberg.catalog.Namespace import org.apache.iceberg.catalog.TableIdentifier import org.apache.iceberg.relocated.com.google.common.base.Preconditions @@ -37,6 +35,7 @@ import org.apache.spark.sql.connector.catalog.View import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.types.StructType +import scala.jdk.CollectionConverters._ case class CreateMaterializedViewExec( catalog: ViewCatalog, From aa582122fced74c4cd23b1f6a0425bbd2f0e9163 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 24 Mar 2026 17:15:15 -0700 Subject: [PATCH 11/22] Fix spotless Scala formatting in v3.5 RewriteViewCommands and parser --- .../sql/catalyst/analysis/RewriteViewCommands.scala | 7 +++++-- .../extensions/IcebergSparkSqlExtensionsParser.scala | 11 ++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala index b75b93a3fa17..3f8168d82b83 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala @@ -41,8 +41,11 @@ import scala.collection.mutable * ResolveSessionCatalog exits early for some v2 View commands, * thus they are pre-substituted here and then handled in ResolveViews */ -case class RewriteViewCommands(spark: SparkSession, materializedViewOptions: Option[MaterializedViewOptions]) - extends Rule[LogicalPlan] with LookupCatalog { +case class RewriteViewCommands( + spark: SparkSession, + materializedViewOptions: Option[MaterializedViewOptions]) + extends Rule[LogicalPlan] + with LookupCatalog { import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala index c13595c08fd1..8be51122c87e 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala @@ -54,7 +54,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) private lazy val substitutor = substitutorCtor.newInstance(SQLConf.get) private lazy val astBuilder = new IcebergSqlExtensionsAstBuilder(delegate) - private lazy final val CREATE_MATERIALIZED_VIEW_PATTERN = "(?i)(CREATE)\\s+MATERIALIZED\\s+(VIEW)".r + private lazy final val CREATE_MATERIALIZED_VIEW_PATTERN = + "(?i)(CREATE)\\s+MATERIALIZED\\s+(VIEW)".r private lazy final val MATERIALIZED_VIEW_STORED_AS_PATTERN = "(?i)STORED AS\\s*'(\\w+)'\\s*".r /** @@ -122,9 +123,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) } .asInstanceOf[LogicalPlan] } else if (isCreateMaterializedView(sqlText)) { - RewriteViewCommands(SparkSession.active, Option(getMaterializedViewOptions(sqlText))).apply( - delegate.parsePlan(getCreateMaterializedViewStatement(sqlText)) - ) + RewriteViewCommands(SparkSession.active, Option(getMaterializedViewOptions(sqlText))) + .apply(delegate.parsePlan(getCreateMaterializedViewStatement(sqlText))) } else { RewriteViewCommands(SparkSession.active, None).apply(delegate.parsePlan(sqlText)) } @@ -169,7 +169,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) } private def getCreateMaterializedViewStatement(sqlText: String): String = { - val createViewSql = CREATE_MATERIALIZED_VIEW_PATTERN.replaceAllIn(sqlText, m => m.group(1) + " " + m.group(2)) + val createViewSql = + CREATE_MATERIALIZED_VIEW_PATTERN.replaceAllIn(sqlText, m => m.group(1) + " " + m.group(2)) MATERIALIZED_VIEW_STORED_AS_PATTERN.replaceAllIn(createViewSql, "") } From 60bd84c5d802e316832486a84a3754e91aed33c6 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 24 Mar 2026 19:28:15 -0700 Subject: [PATCH 12/22] Fix checkstyle violations in core module - Use static imports for assertj Assertions in TestRefreshStateParser - Rename parameter to avoid hidden field in BaseMetastoreViewCatalog --- .../view/BaseMetastoreViewCatalog.java | 4 +- .../iceberg/view/TestRefreshStateParser.java | 74 ++++++++++--------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java b/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java index 174764782ba9..d4411eb033cd 100644 --- a/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java +++ b/core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java @@ -161,8 +161,8 @@ public ViewBuilder withLocation(String newLocation) { } @Override - public ViewBuilder withStorageTableIdentifier(TableIdentifier storageTableIdentifier) { - this.storageTableIdentifier = storageTableIdentifier; + public ViewBuilder withStorageTableIdentifier(TableIdentifier newStorageTableIdentifier) { + this.storageTableIdentifier = newStorageTableIdentifier; return this; } diff --git a/core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java b/core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java index a7fbd8dd44a8..f70add265520 100644 --- a/core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java +++ b/core/src/test/java/org/apache/iceberg/view/TestRefreshStateParser.java @@ -18,9 +18,11 @@ */ package org.apache.iceberg.view; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + import java.util.Arrays; import java.util.Collections; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; public class TestRefreshStateParser { @@ -42,21 +44,21 @@ public void testRoundTripSourceTableState() { String json = RefreshStateParser.toJson(refreshState); RefreshState parsed = RefreshStateParser.fromJson(json); - Assertions.assertThat(parsed.viewVersionId()).isEqualTo(1); - Assertions.assertThat(parsed.refreshStartTimestampMs()).isEqualTo(1573518435000L); - Assertions.assertThat(parsed.sourceStates()).hasSize(1); + assertThat(parsed.viewVersionId()).isEqualTo(1); + assertThat(parsed.refreshStartTimestampMs()).isEqualTo(1573518435000L); + assertThat(parsed.sourceStates()).hasSize(1); SourceState source = parsed.sourceStates().get(0); - Assertions.assertThat(source).isInstanceOf(SourceTableState.class); - Assertions.assertThat(source.type()).isEqualTo("table"); - Assertions.assertThat(source.name()).isEqualTo("events"); - Assertions.assertThat(source.namespace()).containsExactly("default"); - Assertions.assertThat(source.catalog()).isNull(); - Assertions.assertThat(source.uuid()).isEqualTo("d4a10b5c-1e8a-4b72-9d67-3f4a8c9e1b2d"); + assertThat(source).isInstanceOf(SourceTableState.class); + assertThat(source.type()).isEqualTo("table"); + assertThat(source.name()).isEqualTo("events"); + assertThat(source.namespace()).containsExactly("default"); + assertThat(source.catalog()).isNull(); + assertThat(source.uuid()).isEqualTo("d4a10b5c-1e8a-4b72-9d67-3f4a8c9e1b2d"); SourceTableState parsedTable = (SourceTableState) source; - Assertions.assertThat(parsedTable.snapshotId()).isEqualTo(6148331192489823102L); - Assertions.assertThat(parsedTable.ref()).isNull(); + assertThat(parsedTable.snapshotId()).isEqualTo(6148331192489823102L); + assertThat(parsedTable.ref()).isNull(); } @Test @@ -75,17 +77,17 @@ public void testRoundTripSourceViewState() { String json = RefreshStateParser.toJson(refreshState); RefreshState parsed = RefreshStateParser.fromJson(json); - Assertions.assertThat(parsed.sourceStates()).hasSize(1); + assertThat(parsed.sourceStates()).hasSize(1); SourceState source = parsed.sourceStates().get(0); - Assertions.assertThat(source).isInstanceOf(SourceViewState.class); - Assertions.assertThat(source.type()).isEqualTo("view"); - Assertions.assertThat(source.name()).isEqualTo("daily_summary"); - Assertions.assertThat(source.namespace()).containsExactly("analytics", "views"); - Assertions.assertThat(source.catalog()).isEqualTo("other_catalog"); + assertThat(source).isInstanceOf(SourceViewState.class); + assertThat(source.type()).isEqualTo("view"); + assertThat(source.name()).isEqualTo("daily_summary"); + assertThat(source.namespace()).containsExactly("analytics", "views"); + assertThat(source.catalog()).isEqualTo("other_catalog"); SourceViewState parsedView = (SourceViewState) source; - Assertions.assertThat(parsedView.versionId()).isEqualTo(5); + assertThat(parsedView.versionId()).isEqualTo(5); } @Test @@ -102,12 +104,12 @@ public void testRoundTripMixedSourceStates() { String json = RefreshStateParser.toJson(refreshState); RefreshState parsed = RefreshStateParser.fromJson(json); - Assertions.assertThat(parsed.sourceStates()).hasSize(2); - Assertions.assertThat(parsed.sourceStates().get(0)).isInstanceOf(SourceTableState.class); - Assertions.assertThat(parsed.sourceStates().get(1)).isInstanceOf(SourceViewState.class); + assertThat(parsed.sourceStates()).hasSize(2); + assertThat(parsed.sourceStates().get(0)).isInstanceOf(SourceTableState.class); + assertThat(parsed.sourceStates().get(1)).isInstanceOf(SourceViewState.class); SourceTableState parsedTable = (SourceTableState) parsed.sourceStates().get(0); - Assertions.assertThat(parsedTable.ref()).isEqualTo("main"); + assertThat(parsedTable.ref()).isEqualTo("main"); } @Test @@ -127,15 +129,15 @@ public void testParseSpecExample() { RefreshState parsed = RefreshStateParser.fromJson(json); - Assertions.assertThat(parsed.viewVersionId()).isEqualTo(1); - Assertions.assertThat(parsed.refreshStartTimestampMs()).isEqualTo(1573518435000L); - Assertions.assertThat(parsed.sourceStates()).hasSize(1); + assertThat(parsed.viewVersionId()).isEqualTo(1); + assertThat(parsed.refreshStartTimestampMs()).isEqualTo(1573518435000L); + assertThat(parsed.sourceStates()).hasSize(1); SourceTableState tableState = (SourceTableState) parsed.sourceStates().get(0); - Assertions.assertThat(tableState.name()).isEqualTo("events"); - Assertions.assertThat(tableState.namespace()).containsExactly("default"); - Assertions.assertThat(tableState.uuid()).isEqualTo("d4a10b5c-1e8a-4b72-9d67-3f4a8c9e1b2d"); - Assertions.assertThat(tableState.snapshotId()).isEqualTo(6148331192489823102L); + assertThat(tableState.name()).isEqualTo("events"); + assertThat(tableState.namespace()).containsExactly("default"); + assertThat(tableState.uuid()).isEqualTo("d4a10b5c-1e8a-4b72-9d67-3f4a8c9e1b2d"); + assertThat(tableState.snapshotId()).isEqualTo(6148331192489823102L); } @Test @@ -145,8 +147,8 @@ public void testEmptySourceStates() { String json = RefreshStateParser.toJson(refreshState); RefreshState parsed = RefreshStateParser.fromJson(json); - Assertions.assertThat(parsed.sourceStates()).isEmpty(); - Assertions.assertThat(parsed.viewVersionId()).isEqualTo(1); + assertThat(parsed.sourceStates()).isEmpty(); + assertThat(parsed.viewVersionId()).isEqualTo(1); } @Test @@ -159,16 +161,16 @@ public void testSourceTableStateWithRef() { new RefreshState(1, Collections.singletonList(tableState), 1573518435000L); String json = RefreshStateParser.toJson(refreshState); - Assertions.assertThat(json).contains("\"ref\":\"audit_branch\""); + assertThat(json).contains("\"ref\":\"audit_branch\""); RefreshState parsed = RefreshStateParser.fromJson(json); SourceTableState parsedTable = (SourceTableState) parsed.sourceStates().get(0); - Assertions.assertThat(parsedTable.ref()).isEqualTo("audit_branch"); + assertThat(parsedTable.ref()).isEqualTo("audit_branch"); } @Test public void testNullJsonThrows() { - Assertions.assertThatThrownBy(() -> RefreshStateParser.fromJson((String) null)) + assertThatThrownBy(() -> RefreshStateParser.fromJson((String) null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Cannot parse refresh state from null string"); } @@ -187,7 +189,7 @@ public void testUnknownTypeThrows() { + "}]" + "}"; - Assertions.assertThatThrownBy(() -> RefreshStateParser.fromJson(json)) + assertThatThrownBy(() -> RefreshStateParser.fromJson(json)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Unknown source state type: unknown"); } From 4af2a89c1b8b5e08252be41de3d09b4d09e68662 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 24 Mar 2026 19:50:56 -0700 Subject: [PATCH 13/22] Convert v3.5 TestMaterializedViews to JUnit 5 - Use ExtensionsTestBase instead of SparkExtensionsTestBase - Replace JUnit 4 annotations with JUnit 5 (TestTemplate, BeforeEach, AfterEach) - Use ParameterizedTestExtension and Parameters instead of Parameterized - Remove JUnit 4 constructor-based parameter injection --- .../extensions/TestMaterializedViews.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index 3a0aab5f266d..05a2c4ff8b30 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -28,6 +28,8 @@ import java.util.Arrays; import java.util.Map; import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.ParameterizedTestExtension; +import org.apache.iceberg.Parameters; import org.apache.iceberg.TableOperations; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; @@ -52,32 +54,35 @@ import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; import org.apache.spark.sql.connector.catalog.ViewCatalog; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; -public class TestMaterializedViews extends SparkExtensionsTestBase { +@ExtendWith(ParameterizedTestExtension.class) +public class TestMaterializedViews extends ExtensionsTestBase { private static final Namespace NAMESPACE = Namespace.of("default"); private final String tableName = "table"; private final String materializedViewName = "materialized_view"; - @Before + @BeforeEach + @Override public void before() { + super.before(); spark.conf().set("spark.sql.defaultCatalog", catalogName); sql("USE %s", catalogName); sql("CREATE NAMESPACE IF NOT EXISTS %s", NAMESPACE); sql("CREATE TABLE %s (id INT, data STRING)", tableName); } - @After + @AfterEach public void removeTable() { sql("USE %s", catalogName); sql("DROP VIEW IF EXISTS %s", materializedViewName); sql("DROP TABLE IF EXISTS %s", tableName); } - @Parameterized.Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") + @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") public static Object[][] parameters() { Map properties = Maps.newHashMap(SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.properties()); @@ -103,12 +108,7 @@ private static String getTempWarehouseDir() { } } - public TestMaterializedViews( - String catalog, String implementation, Map properties) { - super(catalog, implementation, properties); - } - - @Test + @TestTemplate public void testStorageTableFieldOnViewVersion() { sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); @@ -120,7 +120,7 @@ public void testStorageTableFieldOnViewVersion() { assertThat(view.currentVersion().storageTable().namespace()).isEqualTo(NAMESPACE); } - @Test + @TestTemplate public void testNeverRefreshedMvIsNotFresh() { sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); @@ -133,7 +133,7 @@ public void testNeverRefreshedMvIsNotFresh() { } } - @Test + @TestTemplate public void testReadFromStorageTableWhenFresh() { sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); @@ -153,7 +153,7 @@ public void testReadFromStorageTableWhenFresh() { .isInstanceOf(IllegalStateException.class); } - @Test + @TestTemplate public void testFallbackToViewWhenStale() { sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); @@ -175,7 +175,7 @@ public void testFallbackToViewWhenStale() { .isInstanceOf(NoSuchTableException.class); } - @Test + @TestTemplate public void testStorageTableCreatedBeforeMvMetadata() { sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); @@ -188,7 +188,7 @@ public void testStorageTableCreatedBeforeMvMetadata() { .anySatisfy(row -> assertThat(row[1]).isEqualTo(storageTableName)); } - @Test + @TestTemplate public void testDefaultStorageTableNaming() { sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); @@ -198,7 +198,7 @@ public void testDefaultStorageTableNaming() { .anySatisfy(row -> assertThat(row[1]).isEqualTo(expectedStorageTableName)); } - @Test + @TestTemplate public void testStoredAsClause() { String customTableName = "custom_table_name"; sql( From 345e12ea3bc118d09ef94bc5cb9583163d55e5c9 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 24 Mar 2026 22:40:11 -0700 Subject: [PATCH 14/22] Add message checks to assertThatThrownBy in MV tests Checkstyle requires assertThatThrownBy to include a .hasMessage() check. Applied to both v3.5 and v4.1 TestMaterializedViews. --- .../iceberg/spark/extensions/TestMaterializedViews.java | 6 ++++-- .../iceberg/spark/extensions/TestMaterializedViews.java | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index 05a2c4ff8b30..3a925f98365f 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -150,7 +150,8 @@ public void testReadFromStorageTableWhenFresh() { // Fresh MV: loadView should throw since the engine should use loadTable instead assertThatThrownBy(() -> sparkViewCatalog().loadView(viewIdentifier())) - .isInstanceOf(IllegalStateException.class); + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("fresh"); } @TestTemplate @@ -172,7 +173,8 @@ public void testFallbackToViewWhenStale() { // Stale MV: loadTable should not resolve to the MV's storage table assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(NoSuchTableException.class); + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining(materializedViewName); } @TestTemplate diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index 66c705c2c541..c984cee4f7fd 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -160,7 +160,8 @@ public void testReadFromStorageTableWhenFresh() { // Fresh MV: loadView should throw since the engine should use loadTable instead assertThatThrownBy(() -> sparkViewCatalog().loadView(viewIdentifier())) - .isInstanceOf(IllegalStateException.class); + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("fresh"); } @TestTemplate @@ -182,7 +183,8 @@ public void testFallbackToViewWhenStale() { // Stale MV: loadTable should not resolve to the MV's storage table assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(NoSuchTableException.class); + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining(materializedViewName); } @TestTemplate From e3928de7bb8c3e8f66a52b2519124fb797fbfcf2 Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Tue, 24 Mar 2026 23:55:58 -0700 Subject: [PATCH 15/22] Fix v3.5 TestMaterializedViews to skip configureValidationCatalog Set up validationCatalog manually instead of calling super.before(), matching the v4.1 approach, to avoid IllegalArgumentException from the base class not recognizing InMemoryCatalogWithLocalFileIO. --- .../spark/extensions/TestMaterializedViews.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index 3a925f98365f..79d9cf7208d1 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -68,7 +68,17 @@ public class TestMaterializedViews extends ExtensionsTestBase { @BeforeEach @Override public void before() { - super.before(); + // Set up a simple InMemoryCatalog as validation catalog to avoid base class + // configureValidationCatalog() failing on our custom catalog-impl. + this.validationCatalog = new InMemoryCatalog(); + this.validationNamespaceCatalog = + (org.apache.iceberg.catalog.SupportsNamespaces) validationCatalog; + + spark.conf().set("spark.sql.catalog." + catalogName, implementation); + catalogConfig.forEach( + (key, value) -> spark.conf().set("spark.sql.catalog." + catalogName + "." + key, value)); + + sql("CREATE NAMESPACE IF NOT EXISTS default"); spark.conf().set("spark.sql.defaultCatalog", catalogName); sql("USE %s", catalogName); sql("CREATE NAMESPACE IF NOT EXISTS %s", NAMESPACE); From 3a7cb530c32beb3b20f23d1d681ade1e5149c26c Mon Sep 17 00:00:00 2001 From: Walaa Eldin Moustafa Date: Wed, 25 Mar 2026 21:50:35 -0700 Subject: [PATCH 16/22] Propagate storage table identifier in REST CatalogHandlers.createView --- .../main/java/org/apache/iceberg/rest/CatalogHandlers.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java index 13089fc07ded..b74a87f2fd79 100644 --- a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java @@ -689,6 +689,10 @@ public static LoadViewResponse createView( .withDefaultCatalog(request.viewVersion().defaultCatalog()) .withLocation(request.location()); + if (request.viewVersion().storageTable() != null) { + viewBuilder.withStorageTableIdentifier(request.viewVersion().storageTable()); + } + Set unsupportedRepresentations = request.viewVersion().representations().stream() .filter(r -> !(r instanceof SQLViewRepresentation)) From ec12d45397cde5d9d4bd1e65e3db9080c132145e Mon Sep 17 00:00:00 2001 From: wmoustafa Date: Tue, 23 Jun 2026 11:08:34 -0700 Subject: [PATCH 17/22] Spark: Add REFRESH MATERIALIZED VIEW support in v3.5 and v4.1 extensions --- .../IcebergSparkSqlExtensionsParser.scala | 21 +++ .../RefreshMaterializedViewStatement.scala | 28 ++++ .../v2/ExtendedDataSourceV2Strategy.scala | 4 + .../v2/RefreshMaterializedViewExec.scala | 148 ++++++++++++++++++ .../extensions/TestMaterializedViews.java | 61 ++++++++ .../IcebergSparkSqlExtensionsParser.scala | 21 +++ .../RefreshMaterializedViewStatement.scala | 28 ++++ .../v2/ExtendedDataSourceV2Strategy.scala | 4 + .../v2/RefreshMaterializedViewExec.scala | 148 ++++++++++++++++++ .../extensions/TestMaterializedViews.java | 61 ++++++++ 10 files changed, 524 insertions(+) create mode 100644 spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala create mode 100644 spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala create mode 100644 spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala create mode 100644 spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala index 8be51122c87e..1679822b4180 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala @@ -39,7 +39,9 @@ import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.NonReservedContext import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.QuotedIdentifierContext import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.plans.logical.RefreshMaterializedViewStatement import org.apache.spark.sql.catalyst.trees.Origin +import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.VariableSubstitution import org.apache.spark.sql.types.DataType @@ -125,6 +127,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) } else if (isCreateMaterializedView(sqlText)) { RewriteViewCommands(SparkSession.active, Option(getMaterializedViewOptions(sqlText))) .apply(delegate.parsePlan(getCreateMaterializedViewStatement(sqlText))) + } else if (isRefreshMaterializedView(sqlText)) { + parseRefreshMaterializedView(sqlText) } else { RewriteViewCommands(SparkSession.active, None).apply(delegate.parsePlan(sqlText)) } @@ -180,6 +184,23 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) MaterializedViewOptions(storageTableIdentifier) } + private def isRefreshMaterializedView(sqlText: String): Boolean = { + sqlText.toLowerCase.trim.startsWith("refresh materialized view") + } + + private def parseRefreshMaterializedView(sqlText: String): LogicalPlan = { + val viewName = sqlText.trim + .replaceFirst("(?i)REFRESH\\s+MATERIALIZED\\s+VIEW\\s+", "") + .trim + val spark = SparkSession.active + val catalogAndIdent = + org.apache.iceberg.spark.Spark3Util.catalogAndIdentifier(spark, viewName) + val viewCatalog = + catalogAndIdent.catalog().asInstanceOf[ViewCatalog] + val ident = catalogAndIdent.identifier() + RefreshMaterializedViewStatement(viewCatalog, ident) + } + private def isSnapshotRefDdl(normalized: String): Boolean = { normalized.contains("create branch") || normalized.contains("replace branch") || diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala new file mode 100644 index 000000000000..8de8f2deaa4c --- /dev/null +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.catalyst.plans.logical + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.ViewCatalog + +case class RefreshMaterializedViewStatement(catalog: ViewCatalog, ident: Identifier) + extends LeafCommand { + override def output: Seq[Attribute] = Nil +} diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala index f3a23d006bbb..b4c9037c2fef 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala @@ -41,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField import org.apache.spark.sql.catalyst.plans.logical.DropTag import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.logical.OrderAwareCoalesce +import org.apache.spark.sql.catalyst.plans.logical.RefreshMaterializedViewStatement import org.apache.spark.sql.catalyst.plans.logical.RenameTable import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields @@ -210,6 +211,9 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi case UnsetViewProperties(ResolvedV2View(catalog, ident), propertyKeys, ifExists) => AlterV2ViewUnsetPropertiesExec(catalog, ident, propertyKeys, ifExists) :: Nil + case RefreshMaterializedViewStatement(catalog, ident) => + RefreshMaterializedViewExec(catalog, ident) :: Nil + case _ => Nil } diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala new file mode 100644 index 000000000000..c2bc00755b23 --- /dev/null +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.v2 + +import org.apache.iceberg.catalog.Namespace +import org.apache.iceberg.catalog.TableIdentifier +import org.apache.iceberg.relocated.com.google.common.base.Preconditions +import org.apache.iceberg.spark.SparkCatalog +import org.apache.iceberg.view.RefreshState +import org.apache.iceberg.view.RefreshStateParser +import org.apache.iceberg.view.SourceTableState +import org.apache.iceberg.view.SQLViewRepresentation +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.NoSuchTableException +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.ViewCatalog +import org.apache.spark.sql.functions +import scala.collection.JavaConverters._ + +case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) + extends LeafV2CommandExec { + + override def output: Seq[Attribute] = Nil + + override protected def run(): Seq[InternalRow] = { + val sparkCatalog = catalog.asInstanceOf[SparkCatalog] + val icebergCatalog = sparkCatalog.icebergCatalog() + val icebergViewCatalog = + icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] + val viewId = TableIdentifier.of(Namespace.of(ident.namespace(): _*), ident.name()) + val view = icebergViewCatalog.loadView(viewId) + + val storageTableId = view.currentVersion().storageTable() + Preconditions.checkState( + storageTableId != null, + "Cannot refresh %s: not a materialized view (no storage table)", + ident) + + // Extract the SQL query from the view's representations + val sparkSql = view + .currentVersion() + .representations() + .asScala + .collect { case sql: SQLViewRepresentation if sql.dialect() == "spark" => sql.sql() } + .headOption + .getOrElse(throw new IllegalStateException( + s"Cannot refresh $ident: no Spark SQL representation found")) + + val refreshStartTimestampMs = System.currentTimeMillis() + + // Execute the view's query to get the current result set + val queryResult = session.sql(sparkSql) + + // Discover source tables from the query's logical plan and capture their current state + val sourceStates = collectSourceTableStates(queryResult.queryExecution.analyzed) + + // Build refresh state + val refreshState = new RefreshState( + view.currentVersion().versionId(), + sourceStates.asJava, + refreshStartTimestampMs) + val refreshStateJson = RefreshStateParser.toJson(refreshState) + + // Write results to storage table, replacing existing data + val storageTableRef = String.format( + "%s.%s.%s", + sparkCatalog.name(), + storageTableId.namespace().toString, + storageTableId.name()) + try { + queryResult + .writeTo(storageTableRef) + .option("snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) + .overwrite(functions.lit(true)) + } catch { + case e: NoSuchTableException => + throw new IllegalStateException( + s"Storage table $storageTableRef not found during refresh", + e) + } + + Nil + } + + private def collectSourceTableStates( + plan: org.apache.spark.sql.catalyst.plans.logical.LogicalPlan) + : List[org.apache.iceberg.view.SourceState] = { + val sparkCatalog = catalog.asInstanceOf[SparkCatalog] + val icebergCatalog = + sparkCatalog.icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.Catalog] + val tables = scala.collection.mutable.LinkedHashSet.empty[String] + val states = scala.collection.mutable.ListBuffer.empty[org.apache.iceberg.view.SourceState] + + plan.collectLeaves().foreach { + case r: org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation + if r.catalog.exists(_.name() == sparkCatalog.name()) => + val tableIdent = r.identifier.get + val key = tableIdent.toString + if (!tables.contains(key)) { + tables.add(key) + val icebergId = + TableIdentifier.of(Namespace.of(tableIdent.namespace(): _*), tableIdent.name()) + try { + val table = icebergCatalog.loadTable(icebergId) + val snapshotId = + if (table.currentSnapshot() != null) { + table.currentSnapshot().snapshotId() + } else { + -1L + } + states += new SourceTableState( + icebergId.name(), + icebergId.namespace().levels().toList.asJava, + null, + table.uuid().toString, + snapshotId, + null) + } catch { + case _: Exception => // skip tables we can't load + } + } + case _ => // skip non-iceberg leaves + } + + states.toList + } + + override def simpleString(maxFields: Int): String = { + s"RefreshMaterializedViewExec: ${ident}" + } +} diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index 79d9cf7208d1..7d09ac18ee48 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -221,6 +221,67 @@ public void testStoredAsClause() { assertThat(sql("SHOW TABLES")).anySatisfy(row -> assertThat(row[1]).isEqualTo(customTableName)); } + @TestTemplate + public void testRefreshMaterializedView() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // Refresh the materialized view + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // After refresh, the MV should be fresh and loadable as a table + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Refreshed materialized view should be loadable as a table"); + } + + // Verify the storage table has data + View view = loadIcebergView(); + String storageTableRef = + String.format( + "%s.%s.%s", catalogName, NAMESPACE, view.currentVersion().storageTable().name()); + assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(2); + } + + @TestTemplate + public void testRefreshMaterializedViewUpdatesData() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // First refresh + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // Insert more data + sql("INSERT INTO %s VALUES (3, 'c')", tableName); + + // Before second refresh, the MV should be stale + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); + } catch (NoSuchViewException e) { + fail("Stale materialized view should be loadable as a view"); + } + + // Second refresh + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // After refresh, the MV should be fresh again + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Refreshed materialized view should be loadable as a table"); + } + + // Verify the storage table has all 3 rows + View view = loadIcebergView(); + String storageTableRef = + String.format( + "%s.%s.%s", catalogName, NAMESPACE, view.currentVersion().storageTable().name()); + assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(3); + } + private void simulateRefresh() { View view = loadIcebergView(); org.apache.iceberg.catalog.TableIdentifier storageTableId = diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala index a0070a9ba3fc..f9abea92e0f4 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala @@ -39,7 +39,9 @@ import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.NonReservedContext import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.QuotedIdentifierContext import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.plans.logical.RefreshMaterializedViewStatement import org.apache.spark.sql.catalyst.trees.Origin +import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.VariableSubstitution import org.apache.spark.sql.types.DataType @@ -149,6 +151,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) } else if (isCreateMaterializedView(sqlText)) { RewriteViewCommands(SparkSession.active, Option(getMaterializedViewOptions(sqlText))) .apply(delegate.parsePlan(getCreateMaterializedViewStatement(sqlText))) + } else if (isRefreshMaterializedView(sqlText)) { + parseRefreshMaterializedView(sqlText) } else { RewriteViewCommands(SparkSession.active).apply(delegateParse(sqlText)) } @@ -170,6 +174,23 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) MaterializedViewOptions(storageTableIdentifier) } + private def isRefreshMaterializedView(sqlText: String): Boolean = { + sqlText.toLowerCase.trim.startsWith("refresh materialized view") + } + + private def parseRefreshMaterializedView(sqlText: String): LogicalPlan = { + val viewName = sqlText.trim + .replaceFirst("(?i)REFRESH\\s+MATERIALIZED\\s+VIEW\\s+", "") + .trim + val spark = SparkSession.active + val catalogAndIdent = + org.apache.iceberg.spark.Spark3Util.catalogAndIdentifier(spark, viewName) + val viewCatalog = + catalogAndIdent.catalog().asInstanceOf[ViewCatalog] + val ident = catalogAndIdent.identifier() + RefreshMaterializedViewStatement(viewCatalog, ident) + } + private def isIcebergCommand(sqlText: String): Boolean = { val normalized = sqlText .toLowerCase(Locale.ROOT) diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala new file mode 100644 index 000000000000..8de8f2deaa4c --- /dev/null +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.catalyst.plans.logical + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.ViewCatalog + +case class RefreshMaterializedViewStatement(catalog: ViewCatalog, ident: Identifier) + extends LeafCommand { + override def output: Seq[Attribute] = Nil +} diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala index 3d34e3e9e7c1..0a109e282a08 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala @@ -36,6 +36,7 @@ import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField import org.apache.spark.sql.catalyst.plans.logical.DropTag import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.logical.OrderAwareCoalesce +import org.apache.spark.sql.catalyst.plans.logical.RefreshMaterializedViewStatement import org.apache.spark.sql.catalyst.plans.logical.RenameTable import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields @@ -202,6 +203,9 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi case UnsetViewProperties(ResolvedV2View(catalog, ident), propertyKeys, ifExists) => AlterV2ViewUnsetPropertiesExec(catalog, ident, propertyKeys, ifExists) :: Nil + case RefreshMaterializedViewStatement(catalog, ident) => + RefreshMaterializedViewExec(catalog, ident) :: Nil + case _ => Nil } diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala new file mode 100644 index 000000000000..7eede06997d1 --- /dev/null +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.v2 + +import org.apache.iceberg.catalog.Namespace +import org.apache.iceberg.catalog.TableIdentifier +import org.apache.iceberg.relocated.com.google.common.base.Preconditions +import org.apache.iceberg.spark.SparkCatalog +import org.apache.iceberg.view.RefreshState +import org.apache.iceberg.view.RefreshStateParser +import org.apache.iceberg.view.SourceTableState +import org.apache.iceberg.view.SQLViewRepresentation +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.NoSuchTableException +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.ViewCatalog +import org.apache.spark.sql.functions +import scala.jdk.CollectionConverters._ + +case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) + extends LeafV2CommandExec { + + override def output: Seq[Attribute] = Nil + + override protected def run(): Seq[InternalRow] = { + val sparkCatalog = catalog.asInstanceOf[SparkCatalog] + val icebergCatalog = sparkCatalog.icebergCatalog() + val icebergViewCatalog = + icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] + val viewId = TableIdentifier.of(Namespace.of(ident.namespace(): _*), ident.name()) + val view = icebergViewCatalog.loadView(viewId) + + val storageTableId = view.currentVersion().storageTable() + Preconditions.checkState( + storageTableId != null, + "Cannot refresh %s: not a materialized view (no storage table)", + ident) + + // Extract the SQL query from the view's representations + val sparkSql = view + .currentVersion() + .representations() + .asScala + .collect { case sql: SQLViewRepresentation if sql.dialect() == "spark" => sql.sql() } + .headOption + .getOrElse(throw new IllegalStateException( + s"Cannot refresh $ident: no Spark SQL representation found")) + + val refreshStartTimestampMs = System.currentTimeMillis() + + // Execute the view's query to get the current result set + val queryResult = session.sql(sparkSql) + + // Discover source tables from the query's logical plan and capture their current state + val sourceStates = collectSourceTableStates(queryResult.queryExecution.analyzed) + + // Build refresh state + val refreshState = new RefreshState( + view.currentVersion().versionId(), + sourceStates.asJava, + refreshStartTimestampMs) + val refreshStateJson = RefreshStateParser.toJson(refreshState) + + // Write results to storage table, replacing existing data + val storageTableRef = String.format( + "%s.%s.%s", + sparkCatalog.name(), + storageTableId.namespace().toString, + storageTableId.name()) + try { + queryResult + .writeTo(storageTableRef) + .option("snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) + .overwrite(functions.lit(true)) + } catch { + case e: NoSuchTableException => + throw new IllegalStateException( + s"Storage table $storageTableRef not found during refresh", + e) + } + + Nil + } + + private def collectSourceTableStates( + plan: org.apache.spark.sql.catalyst.plans.logical.LogicalPlan) + : List[org.apache.iceberg.view.SourceState] = { + val sparkCatalog = catalog.asInstanceOf[SparkCatalog] + val icebergCatalog = + sparkCatalog.icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.Catalog] + val tables = scala.collection.mutable.LinkedHashSet.empty[String] + val states = scala.collection.mutable.ListBuffer.empty[org.apache.iceberg.view.SourceState] + + plan.collectLeaves().foreach { + case r: org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation + if r.catalog.exists(_.name() == sparkCatalog.name()) => + val tableIdent = r.identifier.get + val key = tableIdent.toString + if (!tables.contains(key)) { + tables.add(key) + val icebergId = + TableIdentifier.of(Namespace.of(tableIdent.namespace(): _*), tableIdent.name()) + try { + val table = icebergCatalog.loadTable(icebergId) + val snapshotId = + if (table.currentSnapshot() != null) { + table.currentSnapshot().snapshotId() + } else { + -1L + } + states += new SourceTableState( + icebergId.name(), + icebergId.namespace().levels().toList.asJava, + null, + table.uuid().toString, + snapshotId, + null) + } catch { + case _: Exception => // skip tables we can't load + } + } + case _ => // skip non-iceberg leaves + } + + states.toList + } + + override def simpleString(maxFields: Int): String = { + s"RefreshMaterializedViewExec: ${ident}" + } +} diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index c984cee4f7fd..033d1d413689 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -221,6 +221,67 @@ public void testStoredAsClause() { assertThat(sql("SHOW TABLES")).anySatisfy(row -> assertThat(row[1]).isEqualTo(customTableName)); } + @TestTemplate + public void testRefreshMaterializedView() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // Refresh the materialized view + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // After refresh, the MV should be fresh and loadable as a table + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Refreshed materialized view should be loadable as a table"); + } + + // Verify the storage table has data + View view = loadIcebergView(); + String storageTableRef = + String.format( + "%s.%s.%s", catalogName, NAMESPACE, view.currentVersion().storageTable().name()); + assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(2); + } + + @TestTemplate + public void testRefreshMaterializedViewUpdatesData() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // First refresh + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // Insert more data + sql("INSERT INTO %s VALUES (3, 'c')", tableName); + + // Before second refresh, the MV should be stale + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); + } catch (NoSuchViewException e) { + fail("Stale materialized view should be loadable as a view"); + } + + // Second refresh + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // After refresh, the MV should be fresh again + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Refreshed materialized view should be loadable as a table"); + } + + // Verify the storage table has all 3 rows + View view = loadIcebergView(); + String storageTableRef = + String.format( + "%s.%s.%s", catalogName, NAMESPACE, view.currentVersion().storageTable().name()); + assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(3); + } + private void simulateRefresh() { View view = loadIcebergView(); org.apache.iceberg.catalog.TableIdentifier storageTableId = From 2141f0d8fd2eb44efe144df2efbe6df48f652c05 Mon Sep 17 00:00:00 2001 From: wmoustafa Date: Tue, 23 Jun 2026 12:05:54 -0700 Subject: [PATCH 18/22] Address review comments --- .../src/main/java/org/apache/iceberg/spark/SparkCatalog.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index fc9863363fa4..5e9d0b7797d7 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -669,8 +669,7 @@ private boolean isFresh(org.apache.iceberg.view.View view) { tableState.namespace().toArray(new String[0])), tableState.name()); try { - org.apache.iceberg.Table sourceTable = - ((org.apache.iceberg.catalog.Catalog) icebergCatalog()).loadTable(sourceId); + org.apache.iceberg.Table sourceTable = icebergCatalog().loadTable(sourceId); long currentSnapshotId = sourceTable.currentSnapshot() == null ? -1 From 84dfefdaf8d0566c13012046d049ac79ee263faf Mon Sep 17 00:00:00 2001 From: wmoustafa Date: Wed, 5 Aug 2026 23:54:37 -0700 Subject: [PATCH 19/22] Spark: Track and validate nested view state for MV freshness REFRESH MATERIALIZED VIEW previously only recorded SourceTableState for base tables discovered via the analyzed plan's leaves, silently ignoring any nested views the MV's query depends on. isFresh() likewise never checked SourceViewState even though the type already existed. - RefreshMaterializedViewExec: also collect SubqueryAlias nodes across the whole analyzed plan (not just leaves) to discover every nested source view transitively, and record a SourceViewState for each. - SparkCatalog.isFresh: validate SourceViewState entries by comparing the source view's current version id against the recorded one. - Add TestMaterializedViews coverage for both REFRESH capturing nested view state and isFresh detecting staleness from a nested view change. --- .../v2/RefreshMaterializedViewExec.scala | 51 ++++++++-- .../extensions/TestMaterializedViews.java | 97 ++++++++++++++++++- .../apache/iceberg/spark/SparkCatalog.java | 16 +++ .../v2/RefreshMaterializedViewExec.scala | 56 +++++++++-- .../extensions/TestMaterializedViews.java | 97 ++++++++++++++++++- .../apache/iceberg/spark/SparkCatalog.java | 16 +++ 6 files changed, 314 insertions(+), 19 deletions(-) diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala index c2bc00755b23..a17501a9602f 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala @@ -25,17 +25,23 @@ import org.apache.iceberg.spark.SparkCatalog import org.apache.iceberg.view.RefreshState import org.apache.iceberg.view.RefreshStateParser import org.apache.iceberg.view.SourceTableState +import org.apache.iceberg.view.SourceViewState import org.apache.iceberg.view.SQLViewRepresentation import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.NoSuchTableException import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias +import org.apache.spark.sql.connector.catalog.CatalogManager import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.LookupCatalog import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.functions import scala.collection.JavaConverters._ case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) - extends LeafV2CommandExec { + extends LeafV2CommandExec with LookupCatalog { + + protected lazy val catalogManager: CatalogManager = session.sessionState.catalogManager override def output: Seq[Attribute] = Nil @@ -68,8 +74,9 @@ case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) // Execute the view's query to get the current result set val queryResult = session.sql(sparkSql) - // Discover source tables from the query's logical plan and capture their current state - val sourceStates = collectSourceTableStates(queryResult.queryExecution.analyzed) + // Discover source tables and views from the query's logical plan and capture their + // current state + val sourceStates = collectSourceStates(queryResult.queryExecution.analyzed) // Build refresh state val refreshState = new RefreshState( @@ -99,22 +106,23 @@ case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) Nil } - private def collectSourceTableStates( + private def collectSourceStates( plan: org.apache.spark.sql.catalyst.plans.logical.LogicalPlan) : List[org.apache.iceberg.view.SourceState] = { val sparkCatalog = catalog.asInstanceOf[SparkCatalog] val icebergCatalog = sparkCatalog.icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.Catalog] - val tables = scala.collection.mutable.LinkedHashSet.empty[String] + val icebergViewCatalog = + sparkCatalog.icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] + val seen = scala.collection.mutable.LinkedHashSet.empty[String] val states = scala.collection.mutable.ListBuffer.empty[org.apache.iceberg.view.SourceState] plan.collectLeaves().foreach { case r: org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation if r.catalog.exists(_.name() == sparkCatalog.name()) => val tableIdent = r.identifier.get - val key = tableIdent.toString - if (!tables.contains(key)) { - tables.add(key) + val key = "table:" + tableIdent.toString + if (seen.add(key)) { val icebergId = TableIdentifier.of(Namespace.of(tableIdent.namespace(): _*), tableIdent.name()) try { @@ -139,6 +147,33 @@ case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) case _ => // skip non-iceberg leaves } + // Spark's analyzer resolves every view reference into a SubqueryAlias wrapping the + // view's expanded query, including transitively for view-of-view chains, so a single + // pass over the whole plan (not just its leaves) discovers every source view at every + // nesting depth. + plan.collect { case sub: SubqueryAlias => + sub.identifier.qualifier :+ sub.identifier.name + }.collect { + case CatalogAndIdentifier(cat, viewIdent) if cat.name() == sparkCatalog.name() => viewIdent + }.foreach { viewIdent => + val key = "view:" + viewIdent.toString + if (seen.add(key)) { + val icebergId = + TableIdentifier.of(Namespace.of(viewIdent.namespace(): _*), viewIdent.name()) + try { + val view = icebergViewCatalog.loadView(icebergId) + states += new SourceViewState( + icebergId.name(), + icebergId.namespace().levels().toList.asJava, + null, + view.uuid().toString, + view.currentVersion().versionId()) + } catch { + case _: Exception => // not a view, or the view can't be loaded + } + } + } + states.toList } diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index 7d09ac18ee48..f87f708239a7 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -47,6 +47,7 @@ import org.apache.iceberg.view.RefreshState; import org.apache.iceberg.view.RefreshStateParser; import org.apache.iceberg.view.SourceTableState; +import org.apache.iceberg.view.SourceViewState; import org.apache.iceberg.view.View; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; @@ -282,6 +283,96 @@ public void testRefreshMaterializedViewUpdatesData() { assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(3); } + @TestTemplate + public void testRefreshRecordsNestedViewState() { + String sourceViewName = "source_view"; + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE VIEW %s AS SELECT id, data FROM %s", sourceViewName, tableName); + sql( + "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", + materializedViewName, sourceViewName); + + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + View sourceView = loadIcebergView(sourceViewName); + RefreshState refreshState = loadRefreshState(); + + // The refresh state should record both the nested source view and the base table it + // resolves to, since the analyzed query plan fully expands the view chain. + assertThat(refreshState.sourceStates()).hasSize(2); + + SourceViewState viewState = + refreshState.sourceStates().stream() + .filter(SourceViewState.class::isInstance) + .map(SourceViewState.class::cast) + .findFirst() + .orElseGet(() -> fail("Refresh state should record the nested source view")); + assertThat(viewState.name()).isEqualTo(sourceViewName); + assertThat(viewState.namespace()).isEqualTo(Arrays.asList(NAMESPACE.levels())); + assertThat(viewState.uuid()).isEqualTo(sourceView.uuid().toString()); + assertThat(viewState.versionId()).isEqualTo(sourceView.currentVersion().versionId()); + + SourceTableState tableState = + refreshState.sourceStates().stream() + .filter(SourceTableState.class::isInstance) + .map(SourceTableState.class::cast) + .findFirst() + .orElseGet(() -> fail("Refresh state should record the underlying base table")); + assertThat(tableState.name()).isEqualTo(tableName); + + sql("DROP VIEW IF EXISTS %s", sourceViewName); + } + + @TestTemplate + public void testStaleWhenNestedViewChanges() { + String sourceViewName = "source_view"; + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); + sql("CREATE VIEW %s AS SELECT id, data FROM %s WHERE id <= 2", sourceViewName, tableName); + sql( + "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", + materializedViewName, sourceViewName); + + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // Freshly refreshed: loadable as a table + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Refreshed materialized view should be loadable as a table"); + } + + // Replace the nested view's definition without touching the base table. This bumps the + // source view's version but leaves the underlying base table's snapshot unchanged. + sql( + "CREATE OR REPLACE VIEW %s AS SELECT id, data FROM %s WHERE id <= 1", + sourceViewName, tableName); + + // The MV should now be stale because its nested source view changed versions, even + // though the underlying base table's snapshot did not change. + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); + } catch (NoSuchViewException e) { + fail("Materialized view with a stale nested view should be loadable as a view"); + } + assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining(materializedViewName); + + sql("DROP VIEW IF EXISTS %s", sourceViewName); + } + + private RefreshState loadRefreshState() { + View view = loadIcebergView(); + org.apache.iceberg.catalog.TableIdentifier storageTableId = + view.currentVersion().storageTable(); + org.apache.iceberg.Table storageTable = + sparkCatalog().icebergCatalog().loadTable(storageTableId); + String refreshStateJson = + storageTable.currentSnapshot().summary().get(RefreshState.REFRESH_STATE_SUMMARY_KEY); + return RefreshStateParser.fromJson(refreshStateJson); + } + private void simulateRefresh() { View view = loadIcebergView(); org.apache.iceberg.catalog.TableIdentifier storageTableId = @@ -343,9 +434,13 @@ private SparkCatalog sparkCatalog() { } private View loadIcebergView() { + return loadIcebergView(materializedViewName); + } + + private View loadIcebergView(String viewName) { org.apache.iceberg.catalog.ViewCatalog icebergViewCatalog = (org.apache.iceberg.catalog.ViewCatalog) sparkCatalog().icebergCatalog(); - return icebergViewCatalog.loadView(TableIdentifier.of(NAMESPACE, materializedViewName)); + return icebergViewCatalog.loadView(TableIdentifier.of(NAMESPACE, viewName)); } // Required to be public since it is loaded by org.apache.iceberg.CatalogUtil.loadCatalog diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index 118e6bbb7420..4a8777170676 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -680,6 +680,22 @@ private boolean isFresh(org.apache.iceberg.view.View view) { } catch (Exception e) { return false; } + } else if (sourceState instanceof org.apache.iceberg.view.SourceViewState) { + org.apache.iceberg.view.SourceViewState viewState = + (org.apache.iceberg.view.SourceViewState) sourceState; + org.apache.iceberg.catalog.TableIdentifier sourceId = + org.apache.iceberg.catalog.TableIdentifier.of( + org.apache.iceberg.catalog.Namespace.of( + viewState.namespace().toArray(new String[0])), + viewState.name()); + try { + org.apache.iceberg.view.View sourceView = asViewCatalog.loadView(sourceId); + if (sourceView.currentVersion().versionId() != viewState.versionId()) { + return false; + } + } catch (Exception e) { + return false; + } } } diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala index 7eede06997d1..055557319235 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala @@ -25,17 +25,24 @@ import org.apache.iceberg.spark.SparkCatalog import org.apache.iceberg.view.RefreshState import org.apache.iceberg.view.RefreshStateParser import org.apache.iceberg.view.SourceTableState +import org.apache.iceberg.view.SourceViewState import org.apache.iceberg.view.SQLViewRepresentation import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.NoSuchTableException import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias +import org.apache.spark.sql.connector.catalog.CatalogManager import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.LookupCatalog import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.functions import scala.jdk.CollectionConverters._ case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) - extends LeafV2CommandExec { + extends LeafV2CommandExec + with LookupCatalog { + + protected lazy val catalogManager: CatalogManager = session.sessionState.catalogManager override def output: Seq[Attribute] = Nil @@ -68,8 +75,9 @@ case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) // Execute the view's query to get the current result set val queryResult = session.sql(sparkSql) - // Discover source tables from the query's logical plan and capture their current state - val sourceStates = collectSourceTableStates(queryResult.queryExecution.analyzed) + // Discover source tables and views from the query's logical plan and capture their + // current state + val sourceStates = collectSourceStates(queryResult.queryExecution.analyzed) // Build refresh state val refreshState = new RefreshState( @@ -99,22 +107,22 @@ case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) Nil } - private def collectSourceTableStates( - plan: org.apache.spark.sql.catalyst.plans.logical.LogicalPlan) + private def collectSourceStates(plan: org.apache.spark.sql.catalyst.plans.logical.LogicalPlan) : List[org.apache.iceberg.view.SourceState] = { val sparkCatalog = catalog.asInstanceOf[SparkCatalog] val icebergCatalog = sparkCatalog.icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.Catalog] - val tables = scala.collection.mutable.LinkedHashSet.empty[String] + val icebergViewCatalog = + sparkCatalog.icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] + val seen = scala.collection.mutable.LinkedHashSet.empty[String] val states = scala.collection.mutable.ListBuffer.empty[org.apache.iceberg.view.SourceState] plan.collectLeaves().foreach { case r: org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation if r.catalog.exists(_.name() == sparkCatalog.name()) => val tableIdent = r.identifier.get - val key = tableIdent.toString - if (!tables.contains(key)) { - tables.add(key) + val key = "table:" + tableIdent.toString + if (seen.add(key)) { val icebergId = TableIdentifier.of(Namespace.of(tableIdent.namespace(): _*), tableIdent.name()) try { @@ -139,6 +147,36 @@ case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) case _ => // skip non-iceberg leaves } + // Spark's analyzer resolves every view reference into a SubqueryAlias wrapping the + // view's expanded query, including transitively for view-of-view chains, so a single + // pass over the whole plan (not just its leaves) discovers every source view at every + // nesting depth. + plan + .collect { case sub: SubqueryAlias => + sub.identifier.qualifier :+ sub.identifier.name + } + .collect { + case CatalogAndIdentifier(cat, viewIdent) if cat.name() == sparkCatalog.name() => viewIdent + } + .foreach { viewIdent => + val key = "view:" + viewIdent.toString + if (seen.add(key)) { + val icebergId = + TableIdentifier.of(Namespace.of(viewIdent.namespace(): _*), viewIdent.name()) + try { + val view = icebergViewCatalog.loadView(icebergId) + states += new SourceViewState( + icebergId.name(), + icebergId.namespace().levels().toList.asJava, + null, + view.uuid().toString, + view.currentVersion().versionId()) + } catch { + case _: Exception => // not a view, or the view can't be loaded + } + } + } + states.toList } diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index 033d1d413689..167eb5d16cf5 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -47,6 +47,7 @@ import org.apache.iceberg.view.RefreshState; import org.apache.iceberg.view.RefreshStateParser; import org.apache.iceberg.view.SourceTableState; +import org.apache.iceberg.view.SourceViewState; import org.apache.iceberg.view.View; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; @@ -282,6 +283,96 @@ public void testRefreshMaterializedViewUpdatesData() { assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(3); } + @TestTemplate + public void testRefreshRecordsNestedViewState() { + String sourceViewName = "source_view"; + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE VIEW %s AS SELECT id, data FROM %s", sourceViewName, tableName); + sql( + "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", + materializedViewName, sourceViewName); + + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + View sourceView = loadIcebergView(sourceViewName); + RefreshState refreshState = loadRefreshState(); + + // The refresh state should record both the nested source view and the base table it + // resolves to, since the analyzed query plan fully expands the view chain. + assertThat(refreshState.sourceStates()).hasSize(2); + + SourceViewState viewState = + refreshState.sourceStates().stream() + .filter(SourceViewState.class::isInstance) + .map(SourceViewState.class::cast) + .findFirst() + .orElseGet(() -> fail("Refresh state should record the nested source view")); + assertThat(viewState.name()).isEqualTo(sourceViewName); + assertThat(viewState.namespace()).isEqualTo(Arrays.asList(NAMESPACE.levels())); + assertThat(viewState.uuid()).isEqualTo(sourceView.uuid().toString()); + assertThat(viewState.versionId()).isEqualTo(sourceView.currentVersion().versionId()); + + SourceTableState tableState = + refreshState.sourceStates().stream() + .filter(SourceTableState.class::isInstance) + .map(SourceTableState.class::cast) + .findFirst() + .orElseGet(() -> fail("Refresh state should record the underlying base table")); + assertThat(tableState.name()).isEqualTo(tableName); + + sql("DROP VIEW IF EXISTS %s", sourceViewName); + } + + @TestTemplate + public void testStaleWhenNestedViewChanges() { + String sourceViewName = "source_view"; + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); + sql("CREATE VIEW %s AS SELECT id, data FROM %s WHERE id <= 2", sourceViewName, tableName); + sql( + "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", + materializedViewName, sourceViewName); + + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // Freshly refreshed: loadable as a table + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Refreshed materialized view should be loadable as a table"); + } + + // Replace the nested view's definition without touching the base table. This bumps the + // source view's version but leaves the underlying base table's snapshot unchanged. + sql( + "CREATE OR REPLACE VIEW %s AS SELECT id, data FROM %s WHERE id <= 1", + sourceViewName, tableName); + + // The MV should now be stale because its nested source view changed versions, even + // though the underlying base table's snapshot did not change. + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); + } catch (NoSuchViewException e) { + fail("Materialized view with a stale nested view should be loadable as a view"); + } + assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining(materializedViewName); + + sql("DROP VIEW IF EXISTS %s", sourceViewName); + } + + private RefreshState loadRefreshState() { + View view = loadIcebergView(); + org.apache.iceberg.catalog.TableIdentifier storageTableId = + view.currentVersion().storageTable(); + org.apache.iceberg.Table storageTable = + sparkCatalog().icebergCatalog().loadTable(storageTableId); + String refreshStateJson = + storageTable.currentSnapshot().summary().get(RefreshState.REFRESH_STATE_SUMMARY_KEY); + return RefreshStateParser.fromJson(refreshStateJson); + } + private void simulateRefresh() { View view = loadIcebergView(); org.apache.iceberg.catalog.TableIdentifier storageTableId = @@ -343,9 +434,13 @@ private SparkCatalog sparkCatalog() { } private View loadIcebergView() { + return loadIcebergView(materializedViewName); + } + + private View loadIcebergView(String viewName) { org.apache.iceberg.catalog.ViewCatalog icebergViewCatalog = (org.apache.iceberg.catalog.ViewCatalog) sparkCatalog().icebergCatalog(); - return icebergViewCatalog.loadView(TableIdentifier.of(NAMESPACE, materializedViewName)); + return icebergViewCatalog.loadView(TableIdentifier.of(NAMESPACE, viewName)); } // Required to be public since it is loaded by org.apache.iceberg.CatalogUtil.loadCatalog diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index 5e9d0b7797d7..19b57c094abb 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -680,6 +680,22 @@ private boolean isFresh(org.apache.iceberg.view.View view) { } catch (Exception e) { return false; } + } else if (sourceState instanceof org.apache.iceberg.view.SourceViewState) { + org.apache.iceberg.view.SourceViewState viewState = + (org.apache.iceberg.view.SourceViewState) sourceState; + org.apache.iceberg.catalog.TableIdentifier sourceId = + org.apache.iceberg.catalog.TableIdentifier.of( + org.apache.iceberg.catalog.Namespace.of( + viewState.namespace().toArray(new String[0])), + viewState.name()); + try { + org.apache.iceberg.view.View sourceView = asViewCatalog.loadView(sourceId); + if (sourceView.currentVersion().versionId() != viewState.versionId()) { + return false; + } + } catch (Exception e) { + return false; + } } } From 9a2643b88e53f39d3731d2d38c81a3d5bb8af226 Mon Sep 17 00:00:00 2001 From: wmoustafa Date: Tue, 25 Aug 2026 15:31:08 -0700 Subject: [PATCH 20/22] Spark: Limit materialized view support to Spark 4.2 Materialized views are supported only on Spark 4.2, so remove the Spark 3.5 and Spark 4.1 implementations. Spark 4.2 introduces RelationCatalog.loadRelation, a unified table-or-view lookup that lets a catalog route a materialized view to its storage table explicitly. Spark 3.5 and 4.1 have no such entry point, so those versions have to redirect the engine by throwing from loadView, which leaks an unchecked exception to callers that resolve views directly. Supporting a single version keeps the feature on the supported routing path. Removes the per-version implementations under spark/v3.5 and spark/v4.1 and reverts the materialized view changes to the view analysis, planning, and catalog files there. The engine-independent support in api/ and core/ is unchanged and is shared by the Spark 4.2 implementation. --- .../sql/catalyst/analysis/CheckViews.scala | 1 - .../sql/catalyst/analysis/ResolveViews.scala | 1 - .../analysis/RewriteViewCommands.scala | 11 +- .../IcebergSparkSqlExtensionsParser.scala | 46 +- .../RefreshMaterializedViewStatement.scala | 28 - .../logical/views/CreateIcebergView.scala | 6 +- .../v2/CreateMaterializedViewExec.scala | 172 ------ .../datasources/v2/DropV2ViewExec.scala | 32 -- .../v2/ExtendedDataSourceV2Strategy.scala | 33 -- .../v2/RefreshMaterializedViewExec.scala | 183 ------- .../extensions/TestMaterializedViews.java | 495 ------------------ .../iceberg/spark/MaterializedViewUtil.java | 35 -- .../apache/iceberg/spark/SparkCatalog.java | 117 +---- .../spark/source/SparkMaterializedView.java | 57 -- .../iceberg/spark/SparkCatalogConfig.java | 6 +- .../sql/catalyst/analysis/CheckViews.scala | 1 - .../sql/catalyst/analysis/ResolveViews.scala | 1 - .../analysis/RewriteViewCommands.scala | 11 +- .../IcebergSparkSqlExtensionsParser.scala | 44 -- .../RefreshMaterializedViewStatement.scala | 28 - .../logical/views/CreateIcebergView.scala | 2 - .../v2/CreateMaterializedViewExec.scala | 172 ------ .../datasources/v2/DropV2ViewExec.scala | 32 -- .../v2/ExtendedDataSourceV2Strategy.scala | 33 -- .../v2/RefreshMaterializedViewExec.scala | 186 ------- .../extensions/TestMaterializedViews.java | 495 ------------------ .../iceberg/spark/MaterializedViewUtil.java | 35 -- .../apache/iceberg/spark/SparkCatalog.java | 110 +--- .../spark/source/SparkMaterializedView.java | 57 -- .../iceberg/spark/SparkCatalogConfig.java | 12 +- 30 files changed, 11 insertions(+), 2431 deletions(-) delete mode 100644 spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala delete mode 100644 spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala delete mode 100644 spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala delete mode 100644 spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java delete mode 100644 spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java delete mode 100644 spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java delete mode 100644 spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala delete mode 100644 spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala delete mode 100644 spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala delete mode 100644 spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java delete mode 100644 spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java delete mode 100644 spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala index 6f6b41ad11c1..319ab78a5348 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala @@ -49,7 +49,6 @@ object CheckViews extends (LogicalPlan => Unit) { _, replace, _, - _, _) => verifyColumnCount(resolvedIdent, columnAliases, query) SchemaUtils.checkColumnNameDuplication( diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala index 4f304467e31f..b1ebd6cb1266 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala @@ -76,7 +76,6 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look _, _, _, - _, _) if query.resolved && !c.rewritten => val aliased = aliasColumns(query, columnAliases, columnComments) c.copy( diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala index 3f8168d82b83..c47b7d6ef6ac 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala @@ -41,11 +41,7 @@ import scala.collection.mutable * ResolveSessionCatalog exits early for some v2 View commands, * thus they are pre-substituted here and then handled in ResolveViews */ -case class RewriteViewCommands( - spark: SparkSession, - materializedViewOptions: Option[MaterializedViewOptions]) - extends Rule[LogicalPlan] - with LookupCatalog { +case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog { import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ @@ -75,8 +71,7 @@ case class RewriteViewCommands( comment = comment, properties = properties, allowExisting = allowExisting, - replace = replace, - materializedViewOptions = materializedViewOptions) + replace = replace) case view @ ShowViews(UnresolvedNamespace(Seq()), pattern, output) => if (ViewUtil.isViewCatalog(catalogManager.currentCatalog)) { @@ -212,5 +207,3 @@ case class RewriteViewCommands( tempFunctions.toSeq } } - -case class MaterializedViewOptions(storageTableIdentifier: Option[String]) diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala index 1679822b4180..b25333d56787 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala @@ -32,16 +32,13 @@ import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.MaterializedViewOptions import org.apache.spark.sql.catalyst.analysis.RewriteViewCommands import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.NonReservedContext import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.QuotedIdentifierContext import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.plans.logical.RefreshMaterializedViewStatement import org.apache.spark.sql.catalyst.trees.Origin -import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.VariableSubstitution import org.apache.spark.sql.types.DataType @@ -56,9 +53,6 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) private lazy val substitutor = substitutorCtor.newInstance(SQLConf.get) private lazy val astBuilder = new IcebergSqlExtensionsAstBuilder(delegate) - private lazy final val CREATE_MATERIALIZED_VIEW_PATTERN = - "(?i)(CREATE)\\s+MATERIALIZED\\s+(VIEW)".r - private lazy final val MATERIALIZED_VIEW_STORED_AS_PATTERN = "(?i)STORED AS\\s*'(\\w+)'\\s*".r /** * Parse a string to a DataType. @@ -124,13 +118,8 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) if (isIcebergCommand(sqlTextAfterSubstitution)) { parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) } .asInstanceOf[LogicalPlan] - } else if (isCreateMaterializedView(sqlText)) { - RewriteViewCommands(SparkSession.active, Option(getMaterializedViewOptions(sqlText))) - .apply(delegate.parsePlan(getCreateMaterializedViewStatement(sqlText))) - } else if (isRefreshMaterializedView(sqlText)) { - parseRefreshMaterializedView(sqlText) } else { - RewriteViewCommands(SparkSession.active, None).apply(delegate.parsePlan(sqlText)) + RewriteViewCommands(SparkSession.active).apply(delegate.parsePlan(sqlText)) } } @@ -168,39 +157,6 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) SparkProcedures.names().asScala.map("system." + _).exists(normalized.contains) } - private def isCreateMaterializedView(sqlText: String): Boolean = { - sqlText.toLowerCase.contains("create materialized view") - } - - private def getCreateMaterializedViewStatement(sqlText: String): String = { - val createViewSql = - CREATE_MATERIALIZED_VIEW_PATTERN.replaceAllIn(sqlText, m => m.group(1) + " " + m.group(2)) - MATERIALIZED_VIEW_STORED_AS_PATTERN.replaceAllIn(createViewSql, "") - } - - private def getMaterializedViewOptions(sqlText: String): MaterializedViewOptions = { - val storedAsPattern = "(?i)STORED AS\\s*'(\\w+)'\\s*".r - val storageTableIdentifier = storedAsPattern.findFirstMatchIn(sqlText).map(_.group(1)) - MaterializedViewOptions(storageTableIdentifier) - } - - private def isRefreshMaterializedView(sqlText: String): Boolean = { - sqlText.toLowerCase.trim.startsWith("refresh materialized view") - } - - private def parseRefreshMaterializedView(sqlText: String): LogicalPlan = { - val viewName = sqlText.trim - .replaceFirst("(?i)REFRESH\\s+MATERIALIZED\\s+VIEW\\s+", "") - .trim - val spark = SparkSession.active - val catalogAndIdent = - org.apache.iceberg.spark.Spark3Util.catalogAndIdentifier(spark, viewName) - val viewCatalog = - catalogAndIdent.catalog().asInstanceOf[ViewCatalog] - val ident = catalogAndIdent.identifier() - RefreshMaterializedViewStatement(viewCatalog, ident) - } - private def isSnapshotRefDdl(normalized: String): Boolean = { normalized.contains("create branch") || normalized.contains("replace branch") || diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala deleted file mode 100644 index 8de8f2deaa4c..000000000000 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.spark.sql.catalyst.plans.logical - -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.connector.catalog.Identifier -import org.apache.spark.sql.connector.catalog.ViewCatalog - -case class RefreshMaterializedViewStatement(catalog: ViewCatalog, ident: Identifier) - extends LeafCommand { - override def output: Seq[Attribute] = Nil -} diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala index c21f730add94..84a00a4a9a88 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala @@ -19,12 +19,11 @@ package org.apache.spark.sql.catalyst.plans.logical.views import org.apache.spark.sql.catalyst.analysis.AnalysisContext -import org.apache.spark.sql.catalyst.analysis.MaterializedViewOptions import org.apache.spark.sql.catalyst.plans.logical.AnalysisOnlyCommand import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -// Align Iceberg's CreateIcebergView with Spark's CreateViewCommand by extending AnalysisOnlyCommand. -// The command's children are analyzed then hidden, so the optimizer/planner won't traverse the view body. +// Align Iceberg's CreateIcebergView with Spark’s CreateViewCommand by extending AnalysisOnlyCommand. +// The command’s children are analyzed then hidden, so the optimizer/planner won’t traverse the view body. case class CreateIcebergView( child: LogicalPlan, queryText: String, @@ -37,7 +36,6 @@ case class CreateIcebergView( allowExisting: Boolean, replace: Boolean, rewritten: Boolean = false, - materializedViewOptions: Option[MaterializedViewOptions] = None, isAnalyzed: Boolean = false) extends AnalysisOnlyCommand { diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala deleted file mode 100644 index dd6fdcc5f212..000000000000 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.spark.sql.execution.datasources.v2 - -import org.apache.iceberg.catalog.Namespace -import org.apache.iceberg.catalog.TableIdentifier -import org.apache.iceberg.relocated.com.google.common.base.Preconditions -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap -import org.apache.iceberg.spark.MaterializedViewUtil -import org.apache.iceberg.spark.Spark3Util -import org.apache.iceberg.spark.SparkCatalog -import org.apache.iceberg.spark.SparkSchemaUtil -import org.apache.iceberg.spark.source.SparkView -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.connector.catalog.Identifier -import org.apache.spark.sql.connector.catalog.View -import org.apache.spark.sql.connector.catalog.ViewCatalog -import org.apache.spark.sql.connector.expressions.Transform -import org.apache.spark.sql.types.StructType -import scala.collection.JavaConverters._ - -case class CreateMaterializedViewExec( - catalog: ViewCatalog, - ident: Identifier, - queryText: String, - viewSchema: StructType, - columnAliases: Seq[String], - columnComments: Seq[Option[String]], - queryColumnNames: Seq[String], - comment: Option[String], - properties: Map[String, String], - allowExisting: Boolean, - replace: Boolean, - storageTableIdentifier: Option[String]) - extends LeafV2CommandExec { - - override def output: Seq[Attribute] = Nil - - override protected def run(): Seq[InternalRow] = { - - // Check if storageTableIdentifier is provided. If not, generate a default identifier. - val sparkStorageTableIdentifier = storageTableIdentifier match { - case Some(identifier) => { - val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(session, identifier) - val storageTableCatalogName = catalogAndIdentifier.catalog().name() - Preconditions.checkState( - storageTableCatalogName.equals(catalog.name()), - "Storage table identifier must be in the same catalog as the view." + - " Found storage table in catalog: %s, expected: %s.", - Array[Object](storageTableCatalogName, catalog.name())) - catalogAndIdentifier.identifier() - } - case None => MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier(ident) - } - - // Step 1: Create the storage table BEFORE the MV view metadata. - // Per spec: "The storage table must exist and be accessible before the - // materialized view metadata is committed." - // A newly created MV has a storage table with no snapshots until a refresh is performed. - catalog - .asInstanceOf[SparkCatalog] - .createTable( - sparkStorageTableIdentifier, - viewSchema, - new Array[Transform](0), - ImmutableMap.of[String, String]()) - - // Step 2: Create the MV view metadata with a storage-table reference - try { - createView(sparkStorageTableIdentifier.toString) match { - case Some(_) => // success - case None => // allowExisting and view already exists - } - } catch { - case e: Exception => - // If view creation fails, clean up the storage table - try { - catalog.asInstanceOf[SparkCatalog].dropTable(sparkStorageTableIdentifier) - } catch { - case _: Exception => // best effort cleanup - } - throw e - } - - Nil - } - - override def simpleString(maxFields: Int): String = { - s"CreateMaterializedViewExec: ${ident}" - } - - private def createView(storageTableIdentifier: String): Option[View] = { - val icebergSchema = SparkSchemaUtil.convert(viewSchema) - val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name - val currentCatalog = - if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null - val currentNamespace = session.sessionState.catalogManager.currentNamespace - - val engineVersion = "Spark " + org.apache.spark.SPARK_VERSION - val newProperties = properties ++ - comment.map(ViewCatalog.PROP_COMMENT -> _) + - ( - ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion, - ViewCatalog.PROP_ENGINE_VERSION -> engineVersion) + - ("queryColumnNames" -> queryColumnNames.mkString(",")) - - if (replace) { - // CREATE OR REPLACE VIEW - if (catalog.viewExists(ident)) { - catalog.dropView(ident) - } - // FIXME: replaceView API doesn't exist in Spark 3.5 - val viewCatalog = catalog - .asInstanceOf[SparkCatalog] - .icebergCatalog() - .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val icebergView = viewCatalog - .buildView(Spark3Util.identifierToTableIdentifier(ident)) - .withDefaultCatalog(currentCatalog) - .withDefaultNamespace(Namespace.of(currentNamespace: _*)) - .withQuery("spark", queryText) - .withSchema(icebergSchema) - .withLocation(properties.get("location").orNull) - .withProperties(newProperties.asJava) - .withStorageTableIdentifier(TableIdentifier.parse(storageTableIdentifier)) - .create() - Some(new SparkView(catalog.name(), icebergView)) - - } else { - try { - // CREATE VIEW [IF NOT EXISTS] - val viewCatalog = catalog - .asInstanceOf[SparkCatalog] - .icebergCatalog() - .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val icebergView = viewCatalog - .buildView(Spark3Util.identifierToTableIdentifier(ident)) - .withDefaultCatalog(currentCatalog) - .withDefaultNamespace(Namespace.of(currentNamespace: _*)) - .withQuery("spark", queryText) - .withSchema(icebergSchema) - .withLocation(properties.get("location").orNull) - .withProperties(newProperties.asJava) - .withStorageTableIdentifier(TableIdentifier.parse(storageTableIdentifier)) - .create() - Some(new SparkView(catalog.name(), icebergView)) - } catch { - // TODO: Make sure the existing view is also a materialized view - case _: ViewAlreadyExistsException if allowExisting => None - } - } - } - -} diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala index 84c111ae82ef..6dd1188b78e8 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala @@ -18,11 +18,6 @@ */ package org.apache.spark.sql.execution.datasources.v2 -import org.apache.iceberg.catalog.Namespace -import org.apache.iceberg.catalog.TableIdentifier -import org.apache.iceberg.exceptions -import org.apache.iceberg.spark.SparkCatalog -import org.apache.iceberg.view.View import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.NoSuchViewException import org.apache.spark.sql.catalyst.expressions.Attribute @@ -35,33 +30,6 @@ case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boo override lazy val output: Seq[Attribute] = Nil override protected def run(): Seq[InternalRow] = { - // If the catalog is a SparkCatalog, check for materialized view storage table cleanup - catalog match { - case sparkCatalog: SparkCatalog => - val icebergCatalog = sparkCatalog.icebergCatalog() - val icebergViewCatalog = icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - var view: Option[View] = None - try { - val ns = Namespace.of(ident.namespace(): _*) - val viewId = TableIdentifier.of(ns, ident.name()) - view = Some(icebergViewCatalog.loadView(viewId)) - } catch { - case _: exceptions.NoSuchViewException => - if (!ifExists) { - throw new NoSuchViewException(ident) - } - } - // if view is a materialized view, drop the storage table first - view.foreach { v => - val storageTable = v.currentVersion().storageTable() - if (storageTable != null) { - val storageIdent = Identifier.of(storageTable.namespace().levels(), storageTable.name()) - sparkCatalog.dropTable(storageIdent) - } - } - case _ => // not a SparkCatalog, skip MV cleanup - } - val dropped = catalog.dropView(ident) if (!dropped && !ifExists) { throw new NoSuchViewException(ident) diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala index b4c9037c2fef..6b340b72496e 100644 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala +++ b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala @@ -41,7 +41,6 @@ import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField import org.apache.spark.sql.catalyst.plans.logical.DropTag import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.logical.OrderAwareCoalesce -import org.apache.spark.sql.catalyst.plans.logical.RefreshMaterializedViewStatement import org.apache.spark.sql.catalyst.plans.logical.RenameTable import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields @@ -150,35 +149,6 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi allowExisting, replace, _, - Some(materializedViewOptions), - _) => - CreateMaterializedViewExec( - catalog = viewCatalog, - ident = ident, - queryText = queryText, - columnAliases = columnAliases, - columnComments = columnComments, - queryColumnNames = queryColumnNames, - viewSchema = query.schema, - comment = comment, - properties = properties, - allowExisting = allowExisting, - replace = replace, - storageTableIdentifier = materializedViewOptions.storageTableIdentifier) :: Nil - - case CreateIcebergView( - ResolvedIdentifier(viewCatalog: ViewCatalog, ident), - queryText, - query, - columnAliases, - columnComments, - queryColumnNames, - comment, - properties, - allowExisting, - replace, - _, - None, _) => CreateV2ViewExec( catalog = viewCatalog, @@ -211,9 +181,6 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi case UnsetViewProperties(ResolvedV2View(catalog, ident), propertyKeys, ifExists) => AlterV2ViewUnsetPropertiesExec(catalog, ident, propertyKeys, ifExists) :: Nil - case RefreshMaterializedViewStatement(catalog, ident) => - RefreshMaterializedViewExec(catalog, ident) :: Nil - case _ => Nil } diff --git a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala b/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala deleted file mode 100644 index a17501a9602f..000000000000 --- a/spark/v3.5/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.spark.sql.execution.datasources.v2 - -import org.apache.iceberg.catalog.Namespace -import org.apache.iceberg.catalog.TableIdentifier -import org.apache.iceberg.relocated.com.google.common.base.Preconditions -import org.apache.iceberg.spark.SparkCatalog -import org.apache.iceberg.view.RefreshState -import org.apache.iceberg.view.RefreshStateParser -import org.apache.iceberg.view.SourceTableState -import org.apache.iceberg.view.SourceViewState -import org.apache.iceberg.view.SQLViewRepresentation -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.NoSuchTableException -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias -import org.apache.spark.sql.connector.catalog.CatalogManager -import org.apache.spark.sql.connector.catalog.Identifier -import org.apache.spark.sql.connector.catalog.LookupCatalog -import org.apache.spark.sql.connector.catalog.ViewCatalog -import org.apache.spark.sql.functions -import scala.collection.JavaConverters._ - -case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) - extends LeafV2CommandExec with LookupCatalog { - - protected lazy val catalogManager: CatalogManager = session.sessionState.catalogManager - - override def output: Seq[Attribute] = Nil - - override protected def run(): Seq[InternalRow] = { - val sparkCatalog = catalog.asInstanceOf[SparkCatalog] - val icebergCatalog = sparkCatalog.icebergCatalog() - val icebergViewCatalog = - icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val viewId = TableIdentifier.of(Namespace.of(ident.namespace(): _*), ident.name()) - val view = icebergViewCatalog.loadView(viewId) - - val storageTableId = view.currentVersion().storageTable() - Preconditions.checkState( - storageTableId != null, - "Cannot refresh %s: not a materialized view (no storage table)", - ident) - - // Extract the SQL query from the view's representations - val sparkSql = view - .currentVersion() - .representations() - .asScala - .collect { case sql: SQLViewRepresentation if sql.dialect() == "spark" => sql.sql() } - .headOption - .getOrElse(throw new IllegalStateException( - s"Cannot refresh $ident: no Spark SQL representation found")) - - val refreshStartTimestampMs = System.currentTimeMillis() - - // Execute the view's query to get the current result set - val queryResult = session.sql(sparkSql) - - // Discover source tables and views from the query's logical plan and capture their - // current state - val sourceStates = collectSourceStates(queryResult.queryExecution.analyzed) - - // Build refresh state - val refreshState = new RefreshState( - view.currentVersion().versionId(), - sourceStates.asJava, - refreshStartTimestampMs) - val refreshStateJson = RefreshStateParser.toJson(refreshState) - - // Write results to storage table, replacing existing data - val storageTableRef = String.format( - "%s.%s.%s", - sparkCatalog.name(), - storageTableId.namespace().toString, - storageTableId.name()) - try { - queryResult - .writeTo(storageTableRef) - .option("snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) - .overwrite(functions.lit(true)) - } catch { - case e: NoSuchTableException => - throw new IllegalStateException( - s"Storage table $storageTableRef not found during refresh", - e) - } - - Nil - } - - private def collectSourceStates( - plan: org.apache.spark.sql.catalyst.plans.logical.LogicalPlan) - : List[org.apache.iceberg.view.SourceState] = { - val sparkCatalog = catalog.asInstanceOf[SparkCatalog] - val icebergCatalog = - sparkCatalog.icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.Catalog] - val icebergViewCatalog = - sparkCatalog.icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val seen = scala.collection.mutable.LinkedHashSet.empty[String] - val states = scala.collection.mutable.ListBuffer.empty[org.apache.iceberg.view.SourceState] - - plan.collectLeaves().foreach { - case r: org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation - if r.catalog.exists(_.name() == sparkCatalog.name()) => - val tableIdent = r.identifier.get - val key = "table:" + tableIdent.toString - if (seen.add(key)) { - val icebergId = - TableIdentifier.of(Namespace.of(tableIdent.namespace(): _*), tableIdent.name()) - try { - val table = icebergCatalog.loadTable(icebergId) - val snapshotId = - if (table.currentSnapshot() != null) { - table.currentSnapshot().snapshotId() - } else { - -1L - } - states += new SourceTableState( - icebergId.name(), - icebergId.namespace().levels().toList.asJava, - null, - table.uuid().toString, - snapshotId, - null) - } catch { - case _: Exception => // skip tables we can't load - } - } - case _ => // skip non-iceberg leaves - } - - // Spark's analyzer resolves every view reference into a SubqueryAlias wrapping the - // view's expanded query, including transitively for view-of-view chains, so a single - // pass over the whole plan (not just its leaves) discovers every source view at every - // nesting depth. - plan.collect { case sub: SubqueryAlias => - sub.identifier.qualifier :+ sub.identifier.name - }.collect { - case CatalogAndIdentifier(cat, viewIdent) if cat.name() == sparkCatalog.name() => viewIdent - }.foreach { viewIdent => - val key = "view:" + viewIdent.toString - if (seen.add(key)) { - val icebergId = - TableIdentifier.of(Namespace.of(viewIdent.namespace(): _*), viewIdent.name()) - try { - val view = icebergViewCatalog.loadView(icebergId) - states += new SourceViewState( - icebergId.name(), - icebergId.namespace().levels().toList.asJava, - null, - view.uuid().toString, - view.currentVersion().versionId()) - } catch { - case _: Exception => // not a view, or the view can't be loaded - } - } - } - - states.toList - } - - override def simpleString(maxFields: Int): String = { - s"RefreshMaterializedViewExec: ${ident}" - } -} diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java deleted file mode 100644 index f87f708239a7..000000000000 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ /dev/null @@ -1,495 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.spark.extensions; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.api.Assertions.fail; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.Arrays; -import java.util.Map; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.ParameterizedTestExtension; -import org.apache.iceberg.Parameters; -import org.apache.iceberg.TableOperations; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.RuntimeIOException; -import org.apache.iceberg.inmemory.InMemoryCatalog; -import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.io.OutputFile; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; -import org.apache.iceberg.spark.MaterializedViewUtil; -import org.apache.iceberg.spark.SparkCatalog; -import org.apache.iceberg.spark.SparkCatalogConfig; -import org.apache.iceberg.spark.source.SparkMaterializedView; -import org.apache.iceberg.spark.source.SparkView; -import org.apache.iceberg.view.RefreshState; -import org.apache.iceberg.view.RefreshStateParser; -import org.apache.iceberg.view.SourceTableState; -import org.apache.iceberg.view.SourceViewState; -import org.apache.iceberg.view.View; -import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; -import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; -import org.apache.spark.sql.connector.catalog.CatalogPlugin; -import org.apache.spark.sql.connector.catalog.Identifier; -import org.apache.spark.sql.connector.catalog.TableCatalog; -import org.apache.spark.sql.connector.catalog.ViewCatalog; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.api.extension.ExtendWith; - -@ExtendWith(ParameterizedTestExtension.class) -public class TestMaterializedViews extends ExtensionsTestBase { - private static final Namespace NAMESPACE = Namespace.of("default"); - private final String tableName = "table"; - private final String materializedViewName = "materialized_view"; - - @BeforeEach - @Override - public void before() { - // Set up a simple InMemoryCatalog as validation catalog to avoid base class - // configureValidationCatalog() failing on our custom catalog-impl. - this.validationCatalog = new InMemoryCatalog(); - this.validationNamespaceCatalog = - (org.apache.iceberg.catalog.SupportsNamespaces) validationCatalog; - - spark.conf().set("spark.sql.catalog." + catalogName, implementation); - catalogConfig.forEach( - (key, value) -> spark.conf().set("spark.sql.catalog." + catalogName + "." + key, value)); - - sql("CREATE NAMESPACE IF NOT EXISTS default"); - spark.conf().set("spark.sql.defaultCatalog", catalogName); - sql("USE %s", catalogName); - sql("CREATE NAMESPACE IF NOT EXISTS %s", NAMESPACE); - sql("CREATE TABLE %s (id INT, data STRING)", tableName); - } - - @AfterEach - public void removeTable() { - sql("USE %s", catalogName); - sql("DROP VIEW IF EXISTS %s", materializedViewName); - sql("DROP TABLE IF EXISTS %s", tableName); - } - - @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") - public static Object[][] parameters() { - Map properties = - Maps.newHashMap(SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.properties()); - properties.put(CatalogProperties.WAREHOUSE_LOCATION, "file:" + getTempWarehouseDir()); - properties.put(CatalogProperties.CATALOG_IMPL, InMemoryCatalogWithLocalFileIO.class.getName()); - return new Object[][] { - { - SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.catalogName(), - SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.implementation(), - properties - } - }; - } - - private static String getTempWarehouseDir() { - try { - File tempDir = Files.createTempDirectory("warehouse-").toFile(); - tempDir.deleteOnExit(); - return tempDir.getAbsolutePath(); - - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @TestTemplate - public void testStorageTableFieldOnViewVersion() { - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - View view = loadIcebergView(); - // storage-table should be set on the view version, not as a property - assertThat(view.currentVersion().storageTable()).isNotNull(); - assertThat(view.currentVersion().storageTable().name()) - .isEqualTo(materializedViewName + "__storage"); - assertThat(view.currentVersion().storageTable().namespace()).isEqualTo(NAMESPACE); - } - - @TestTemplate - public void testNeverRefreshedMvIsNotFresh() { - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - // A newly created MV has no snapshots on its storage table, so it's not fresh. - // loadView should succeed (returns stale view) - try { - assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); - } catch (NoSuchViewException e) { - fail("Materialized view not found"); - } - } - - @TestTemplate - public void testReadFromStorageTableWhenFresh() { - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - simulateRefresh(); - - // Fresh MV: loadTable should return SparkMaterializedView - try { - assertThat(sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(SparkMaterializedView.class); - } catch (NoSuchTableException e) { - fail("Fresh materialized view should be loadable as a table"); - } - - // Fresh MV: loadView should throw since the engine should use loadTable instead - assertThatThrownBy(() -> sparkViewCatalog().loadView(viewIdentifier())) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("fresh"); - } - - @TestTemplate - public void testFallbackToViewWhenStale() { - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - simulateRefresh(); - - // Insert more data to invalidate the refresh - sql("INSERT INTO %s VALUES (3, 'c')", tableName); - - // Stale MV: loadView should return SparkView (falls back to query execution) - try { - assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); - } catch (NoSuchViewException e) { - fail("Stale materialized view should be loadable as a view"); - } - - // Stale MV: loadTable should not resolve to the MV's storage table - assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(NoSuchTableException.class) - .hasMessageContaining(materializedViewName); - } - - @TestTemplate - public void testStorageTableCreatedBeforeMvMetadata() { - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - // The storage table should exist - String storageTableName = - MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier( - Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName)) - .name(); - assertThat(sql("SHOW TABLES")) - .anySatisfy(row -> assertThat(row[1]).isEqualTo(storageTableName)); - } - - @TestTemplate - public void testDefaultStorageTableNaming() { - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - // Default naming should be __storage - String expectedStorageTableName = materializedViewName + "__storage"; - assertThat(sql("SHOW TABLES")) - .anySatisfy(row -> assertThat(row[1]).isEqualTo(expectedStorageTableName)); - } - - @TestTemplate - public void testStoredAsClause() { - String customTableName = "custom_table_name"; - sql( - "CREATE MATERIALIZED VIEW %s STORED AS '%s' AS SELECT id, data FROM %s", - materializedViewName, customTableName, tableName); - - // Assert that the storage table with the custom name is in the list of tables - assertThat(sql("SHOW TABLES")).anySatisfy(row -> assertThat(row[1]).isEqualTo(customTableName)); - } - - @TestTemplate - public void testRefreshMaterializedView() { - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - // Refresh the materialized view - sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); - - // After refresh, the MV should be fresh and loadable as a table - try { - assertThat(sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(SparkMaterializedView.class); - } catch (NoSuchTableException e) { - fail("Refreshed materialized view should be loadable as a table"); - } - - // Verify the storage table has data - View view = loadIcebergView(); - String storageTableRef = - String.format( - "%s.%s.%s", catalogName, NAMESPACE, view.currentVersion().storageTable().name()); - assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(2); - } - - @TestTemplate - public void testRefreshMaterializedViewUpdatesData() { - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - // First refresh - sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); - - // Insert more data - sql("INSERT INTO %s VALUES (3, 'c')", tableName); - - // Before second refresh, the MV should be stale - try { - assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); - } catch (NoSuchViewException e) { - fail("Stale materialized view should be loadable as a view"); - } - - // Second refresh - sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); - - // After refresh, the MV should be fresh again - try { - assertThat(sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(SparkMaterializedView.class); - } catch (NoSuchTableException e) { - fail("Refreshed materialized view should be loadable as a table"); - } - - // Verify the storage table has all 3 rows - View view = loadIcebergView(); - String storageTableRef = - String.format( - "%s.%s.%s", catalogName, NAMESPACE, view.currentVersion().storageTable().name()); - assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(3); - } - - @TestTemplate - public void testRefreshRecordsNestedViewState() { - String sourceViewName = "source_view"; - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); - sql("CREATE VIEW %s AS SELECT id, data FROM %s", sourceViewName, tableName); - sql( - "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", - materializedViewName, sourceViewName); - - sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); - - View sourceView = loadIcebergView(sourceViewName); - RefreshState refreshState = loadRefreshState(); - - // The refresh state should record both the nested source view and the base table it - // resolves to, since the analyzed query plan fully expands the view chain. - assertThat(refreshState.sourceStates()).hasSize(2); - - SourceViewState viewState = - refreshState.sourceStates().stream() - .filter(SourceViewState.class::isInstance) - .map(SourceViewState.class::cast) - .findFirst() - .orElseGet(() -> fail("Refresh state should record the nested source view")); - assertThat(viewState.name()).isEqualTo(sourceViewName); - assertThat(viewState.namespace()).isEqualTo(Arrays.asList(NAMESPACE.levels())); - assertThat(viewState.uuid()).isEqualTo(sourceView.uuid().toString()); - assertThat(viewState.versionId()).isEqualTo(sourceView.currentVersion().versionId()); - - SourceTableState tableState = - refreshState.sourceStates().stream() - .filter(SourceTableState.class::isInstance) - .map(SourceTableState.class::cast) - .findFirst() - .orElseGet(() -> fail("Refresh state should record the underlying base table")); - assertThat(tableState.name()).isEqualTo(tableName); - - sql("DROP VIEW IF EXISTS %s", sourceViewName); - } - - @TestTemplate - public void testStaleWhenNestedViewChanges() { - String sourceViewName = "source_view"; - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); - sql("CREATE VIEW %s AS SELECT id, data FROM %s WHERE id <= 2", sourceViewName, tableName); - sql( - "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", - materializedViewName, sourceViewName); - - sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); - - // Freshly refreshed: loadable as a table - try { - assertThat(sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(SparkMaterializedView.class); - } catch (NoSuchTableException e) { - fail("Refreshed materialized view should be loadable as a table"); - } - - // Replace the nested view's definition without touching the base table. This bumps the - // source view's version but leaves the underlying base table's snapshot unchanged. - sql( - "CREATE OR REPLACE VIEW %s AS SELECT id, data FROM %s WHERE id <= 1", - sourceViewName, tableName); - - // The MV should now be stale because its nested source view changed versions, even - // though the underlying base table's snapshot did not change. - try { - assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); - } catch (NoSuchViewException e) { - fail("Materialized view with a stale nested view should be loadable as a view"); - } - assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(NoSuchTableException.class) - .hasMessageContaining(materializedViewName); - - sql("DROP VIEW IF EXISTS %s", sourceViewName); - } - - private RefreshState loadRefreshState() { - View view = loadIcebergView(); - org.apache.iceberg.catalog.TableIdentifier storageTableId = - view.currentVersion().storageTable(); - org.apache.iceberg.Table storageTable = - sparkCatalog().icebergCatalog().loadTable(storageTableId); - String refreshStateJson = - storageTable.currentSnapshot().summary().get(RefreshState.REFRESH_STATE_SUMMARY_KEY); - return RefreshStateParser.fromJson(refreshStateJson); - } - - private void simulateRefresh() { - View view = loadIcebergView(); - org.apache.iceberg.catalog.TableIdentifier storageTableId = - view.currentVersion().storageTable(); - - // Get the base table's current snapshot ID - long baseSnapshotId = - (Long) - sql( - "SELECT snapshot_id FROM %s.%s.%s.snapshots ORDER BY committed_at DESC LIMIT 1", - catalogName, NAMESPACE, tableName) - .get(0)[0]; - - // Build refresh state matching the current view version and source table state - RefreshState refreshState = - new RefreshState( - view.currentVersion().versionId(), - Arrays.asList( - new SourceTableState( - tableName, - Arrays.asList(NAMESPACE.levels()), - null, - "test-uuid", - baseSnapshotId, - null)), - System.currentTimeMillis()); - String refreshStateJson = RefreshStateParser.toJson(refreshState); - - // Write data to storage table with refresh-state in the snapshot summary - String storageTableRef = - String.format("%s.%s.%s", catalogName, NAMESPACE, storageTableId.name()); - try { - spark - .sql(String.format("SELECT id, data FROM %s.%s.%s", catalogName, NAMESPACE, tableName)) - .writeTo(storageTableRef) - .option("snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) - .append(); - } catch (NoSuchTableException e) { - throw new RuntimeException("Storage table not found during simulated refresh", e); - } - } - - private ViewCatalog sparkViewCatalog() { - CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); - return (ViewCatalog) catalogPlugin; - } - - private TableCatalog sparkTableCatalog() { - CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); - return (TableCatalog) catalogPlugin; - } - - private Identifier viewIdentifier() { - return Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName); - } - - private SparkCatalog sparkCatalog() { - return (SparkCatalog) spark.sessionState().catalogManager().catalog(catalogName); - } - - private View loadIcebergView() { - return loadIcebergView(materializedViewName); - } - - private View loadIcebergView(String viewName) { - org.apache.iceberg.catalog.ViewCatalog icebergViewCatalog = - (org.apache.iceberg.catalog.ViewCatalog) sparkCatalog().icebergCatalog(); - return icebergViewCatalog.loadView(TableIdentifier.of(NAMESPACE, viewName)); - } - - // Required to be public since it is loaded by org.apache.iceberg.CatalogUtil.loadCatalog - public static class InMemoryCatalogWithLocalFileIO extends InMemoryCatalog { - private FileIO localFileIO; - - @Override - public void initialize(String name, Map properties) { - super.initialize(name, properties); - localFileIO = new LocalFileIO(); - } - - @Override - protected TableOperations newTableOps(TableIdentifier tableIdentifier) { - return new InMemoryTableOperations(localFileIO, tableIdentifier); - } - - @Override - protected InMemoryCatalog.InMemoryViewOperations newViewOps(TableIdentifier identifier) { - return new InMemoryViewOperations(localFileIO, identifier); - } - } - - private static class LocalFileIO implements FileIO { - - private static String stripFilePrefix(String path) { - return path.startsWith("file:") ? path.substring(5) : path; - } - - @Override - public InputFile newInputFile(String path) { - return org.apache.iceberg.Files.localInput(stripFilePrefix(path)); - } - - @Override - public OutputFile newOutputFile(String path) { - String stripped = stripFilePrefix(path); - java.io.File parent = new java.io.File(stripped).getParentFile(); - if (!parent.isDirectory()) { - parent.mkdirs(); - } - return org.apache.iceberg.Files.localOutput(stripped); - } - - @Override - public void deleteFile(String path) { - if (!new File(stripFilePrefix(path)).delete()) { - throw new RuntimeIOException("Failed to delete file: " + path); - } - } - } -} diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java deleted file mode 100644 index a30c5f671176..000000000000 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.spark; - -import org.apache.spark.sql.connector.catalog.Identifier; - -public class MaterializedViewUtil { - - private MaterializedViewUtil() {} - - private static final String MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX = "__storage"; - - public static Identifier getDefaultMaterializedViewStorageTableIdentifier( - Identifier viewIdentifier) { - return Identifier.of( - viewIdentifier.namespace(), - viewIdentifier.name() + MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX); - } -} diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index 4a8777170676..31e6874c6739 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -63,7 +63,6 @@ import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.spark.actions.SparkActions; import org.apache.iceberg.spark.source.SparkChangelogTable; -import org.apache.iceberg.spark.source.SparkMaterializedView; import org.apache.iceberg.spark.source.SparkTable; import org.apache.iceberg.spark.source.SparkView; import org.apache.iceberg.spark.source.StagedSparkTable; @@ -594,14 +593,7 @@ public View loadView(Identifier ident) throws NoSuchViewException { if (null != asViewCatalog) { try { org.apache.iceberg.view.View view = asViewCatalog.loadView(buildIdentifier(ident)); - // Check if the view is a materialized view. If it is, and storage table is fresh, throw - // IllegalStateException - if (isMaterializedView(view) && isFresh(view)) { - throw new IllegalStateException( - "Materialized view is fresh. loadTable should be attempted instead."); - } else { - return new SparkView(catalogName, view); - } + return new SparkView(catalogName, view); } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { throw new NoSuchViewException(ident); } @@ -610,98 +602,6 @@ public View loadView(Identifier ident) throws NoSuchViewException { throw new NoSuchViewException(ident); } - private boolean isMaterializedView(org.apache.iceberg.view.View view) { - return view.currentVersion().storageTable() != null; - } - - private org.apache.iceberg.catalog.TableIdentifier getStorageTableId( - org.apache.iceberg.view.View view) { - org.apache.iceberg.catalog.TableIdentifier storageTable = view.currentVersion().storageTable(); - Preconditions.checkState( - storageTable != null, "Storage table identifier is not set for materialized view."); - return storageTable; - } - - private Table loadStorageTable(org.apache.iceberg.view.View view) { - org.apache.iceberg.catalog.TableIdentifier storageTableId = getStorageTableId(view); - try { - Identifier sparkIdent = - Identifier.of(storageTableId.namespace().levels(), storageTableId.name()); - return loadTable(sparkIdent); - } catch (NoSuchTableException e) { - throw new IllegalStateException("Unable to load storage table for materialized view.", e); - } - } - - private boolean isFresh(org.apache.iceberg.view.View view) { - Table sparkStorageTable = loadStorageTable(view); - org.apache.iceberg.Table storageTable = ((SparkTable) sparkStorageTable).table(); - if (storageTable.currentSnapshot() == null) { - return false; - } - - String refreshStateJson = - storageTable - .currentSnapshot() - .summary() - .get(org.apache.iceberg.view.RefreshState.REFRESH_STATE_SUMMARY_KEY); - if (refreshStateJson == null) { - return false; - } - - org.apache.iceberg.view.RefreshState refreshState = - org.apache.iceberg.view.RefreshStateParser.fromJson(refreshStateJson); - - // If the refresh was performed against a different view version, the MV is not fresh - if (refreshState.viewVersionId() != view.currentVersion().versionId()) { - return false; - } - - // Check each source table state against the current state - for (org.apache.iceberg.view.SourceState sourceState : refreshState.sourceStates()) { - if (sourceState instanceof org.apache.iceberg.view.SourceTableState) { - org.apache.iceberg.view.SourceTableState tableState = - (org.apache.iceberg.view.SourceTableState) sourceState; - org.apache.iceberg.catalog.TableIdentifier sourceId = - org.apache.iceberg.catalog.TableIdentifier.of( - org.apache.iceberg.catalog.Namespace.of( - tableState.namespace().toArray(new String[0])), - tableState.name()); - try { - org.apache.iceberg.Table sourceTable = - ((org.apache.iceberg.catalog.Catalog) icebergCatalog()).loadTable(sourceId); - long currentSnapshotId = - sourceTable.currentSnapshot() == null - ? -1 - : sourceTable.currentSnapshot().snapshotId(); - if (currentSnapshotId != tableState.snapshotId()) { - return false; - } - } catch (Exception e) { - return false; - } - } else if (sourceState instanceof org.apache.iceberg.view.SourceViewState) { - org.apache.iceberg.view.SourceViewState viewState = - (org.apache.iceberg.view.SourceViewState) sourceState; - org.apache.iceberg.catalog.TableIdentifier sourceId = - org.apache.iceberg.catalog.TableIdentifier.of( - org.apache.iceberg.catalog.Namespace.of( - viewState.namespace().toArray(new String[0])), - viewState.name()); - try { - org.apache.iceberg.view.View sourceView = asViewCatalog.loadView(sourceId); - if (sourceView.currentVersion().versionId() != viewState.versionId()) { - return false; - } - } catch (Exception e) { - return false; - } - } - } - - return true; - } - @Override public View createView( Identifier ident, @@ -988,26 +888,11 @@ private static void checkNotPathIdentifier(Identifier identifier, String method) } } - // TODO Remove @SuppressWarnings - @SuppressWarnings("checkstyle:CyclomaticComplexity") private Table load(Identifier ident) { if (isPathIdentifier(ident)) { return loadFromPathIdentifier((PathIdentifier) ident); } - // Check if materialized view. If fresh, return the SparkMaterializedView. - if (null != asViewCatalog) { - try { - org.apache.iceberg.view.View view = asViewCatalog.loadView(buildIdentifier(ident)); - if (isMaterializedView(view) && isFresh(view)) { - Table storageTable = loadStorageTable(view); - return new SparkMaterializedView(catalogName, view, storageTable); - } - } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { - // Ignore. Just process as a normal table. - } - } - try { org.apache.iceberg.Table table = icebergCatalog.loadTable(buildIdentifier(ident)); return new SparkTable(table, !cacheEnabled); diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java deleted file mode 100644 index 0c01ae449292..000000000000 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.spark.source; - -import java.util.Set; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; -import org.apache.iceberg.view.View; -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.connector.catalog.SupportsRead; -import org.apache.spark.sql.connector.catalog.Table; -import org.apache.spark.sql.connector.catalog.TableCapability; -import org.apache.spark.sql.connector.read.ScanBuilder; -import org.apache.spark.sql.util.CaseInsensitiveStringMap; - -public class SparkMaterializedView extends SparkView implements SupportsRead { - private final Table storageTable; - private SparkSession lazySpark; - - public SparkMaterializedView(String catalogName, View icebergView, Table storageTable) { - super(catalogName, icebergView); - this.storageTable = storageTable; - } - - @Override - public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) { - return ((SupportsRead) storageTable).newScanBuilder(options); - } - - private SparkSession sparkSession() { - if (lazySpark == null) { - this.lazySpark = SparkSession.active(); - } - - return lazySpark; - } - - @Override - public Set capabilities() { - return ImmutableSet.of(TableCapability.BATCH_READ); - } -} diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java index 38802fb7f76e..2350aab09b64 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java @@ -68,11 +68,7 @@ public enum SparkCatalogConfig { SPARK_WITH_HIVE_VIEWS( "spark_hive_with_views", SparkCatalog.class.getName(), - ImmutableMap.of("type", "hive", "default-namespace", "default", "cache-enabled", "false")), - SPARK_WITH_MATERIALIZED_VIEWS( - "spark_with_materialized_views", - SparkCatalog.class.getName(), - ImmutableMap.of("default-namespace", "default", "cache-enabled", "false")); + ImmutableMap.of("type", "hive", "default-namespace", "default", "cache-enabled", "false")); private final String catalogName; private final String implementation; diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala index 1ad164752615..5ad4b9c01409 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckViews.scala @@ -49,7 +49,6 @@ object CheckViews extends (LogicalPlan => Unit) { _, replace, _, - _, _) => verifyColumnCount(resolvedIdent, columnAliases, query) SchemaUtils.checkColumnNameDuplication( diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala index 4f8d5335674f..83e501257ced 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveViews.scala @@ -76,7 +76,6 @@ case class ResolveViews(spark: SparkSession) extends Rule[LogicalPlan] with Look _, _, _, - _, _) if query.resolved && !c.rewritten => val aliased = aliasColumns(query, columnAliases, columnComments) c.copy( diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala index 3ffef317af42..ac0f75c422d1 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala @@ -40,11 +40,7 @@ import scala.collection.mutable * ResolveSessionCatalog exits early for some v2 View commands, * thus they are pre-substituted here and then handled in ResolveViews */ -case class RewriteViewCommands( - spark: SparkSession, - materializedViewOptions: Option[MaterializedViewOptions] = None) - extends Rule[LogicalPlan] - with LookupCatalog { +case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog { import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ @@ -76,8 +72,7 @@ case class RewriteViewCommands( comment = comment, properties = properties, allowExisting = allowExisting, - replace = replace, - materializedViewOptions = materializedViewOptions) + replace = replace) case view @ ShowViews(CurrentNamespace, pattern, output) => if (ViewUtil.isViewCatalog(catalogManager.currentCatalog)) { @@ -213,5 +208,3 @@ case class RewriteViewCommands( tempFunctions.toSeq } } - -case class MaterializedViewOptions(storageTableIdentifier: Option[String]) diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala index f9abea92e0f4..7c737f0513ed 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala @@ -31,7 +31,6 @@ import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.MaterializedViewOptions import org.apache.spark.sql.catalyst.analysis.RewriteViewCommands import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.ParameterContext @@ -39,9 +38,7 @@ import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.NonReservedContext import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.QuotedIdentifierContext import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.plans.logical.RefreshMaterializedViewStatement import org.apache.spark.sql.catalyst.trees.Origin -import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.VariableSubstitution import org.apache.spark.sql.types.DataType @@ -56,9 +53,6 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) private lazy val substitutor = substitutorCtor.newInstance(SQLConf.get) private lazy val astBuilder = new IcebergSqlExtensionsAstBuilder(delegate) - private lazy final val CREATE_MATERIALIZED_VIEW_PATTERN = - "(?i)(CREATE)\\s+MATERIALIZED\\s+(VIEW)".r - private lazy final val MATERIALIZED_VIEW_STORED_AS_PATTERN = "(?i)STORED AS\\s*'(\\w+)'\\s*".r /** * Parse a string to a DataType. @@ -148,49 +142,11 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) if (isIcebergCommand(sqlTextAfterSubstitution)) { parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) } .asInstanceOf[LogicalPlan] - } else if (isCreateMaterializedView(sqlText)) { - RewriteViewCommands(SparkSession.active, Option(getMaterializedViewOptions(sqlText))) - .apply(delegate.parsePlan(getCreateMaterializedViewStatement(sqlText))) - } else if (isRefreshMaterializedView(sqlText)) { - parseRefreshMaterializedView(sqlText) } else { RewriteViewCommands(SparkSession.active).apply(delegateParse(sqlText)) } } - private def isCreateMaterializedView(sqlText: String): Boolean = { - sqlText.toLowerCase.contains("create materialized view") - } - - private def getCreateMaterializedViewStatement(sqlText: String): String = { - val createViewSql = - CREATE_MATERIALIZED_VIEW_PATTERN.replaceAllIn(sqlText, m => m.group(1) + " " + m.group(2)) - MATERIALIZED_VIEW_STORED_AS_PATTERN.replaceAllIn(createViewSql, "") - } - - private def getMaterializedViewOptions(sqlText: String): MaterializedViewOptions = { - val storedAsPattern = "(?i)STORED AS\\s*'(\\w+)'\\s*".r - val storageTableIdentifier = storedAsPattern.findFirstMatchIn(sqlText).map(_.group(1)) - MaterializedViewOptions(storageTableIdentifier) - } - - private def isRefreshMaterializedView(sqlText: String): Boolean = { - sqlText.toLowerCase.trim.startsWith("refresh materialized view") - } - - private def parseRefreshMaterializedView(sqlText: String): LogicalPlan = { - val viewName = sqlText.trim - .replaceFirst("(?i)REFRESH\\s+MATERIALIZED\\s+VIEW\\s+", "") - .trim - val spark = SparkSession.active - val catalogAndIdent = - org.apache.iceberg.spark.Spark3Util.catalogAndIdentifier(spark, viewName) - val viewCatalog = - catalogAndIdent.catalog().asInstanceOf[ViewCatalog] - val ident = catalogAndIdent.identifier() - RefreshMaterializedViewStatement(viewCatalog, ident) - } - private def isIcebergCommand(sqlText: String): Boolean = { val normalized = sqlText .toLowerCase(Locale.ROOT) diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala deleted file mode 100644 index 8de8f2deaa4c..000000000000 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.spark.sql.catalyst.plans.logical - -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.connector.catalog.Identifier -import org.apache.spark.sql.connector.catalog.ViewCatalog - -case class RefreshMaterializedViewStatement(catalog: ViewCatalog, ident: Identifier) - extends LeafCommand { - override def output: Seq[Attribute] = Nil -} diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala index 3e11d18b45fe..84a00a4a9a88 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala @@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.plans.logical.views import org.apache.spark.sql.catalyst.analysis.AnalysisContext -import org.apache.spark.sql.catalyst.analysis.MaterializedViewOptions import org.apache.spark.sql.catalyst.plans.logical.AnalysisOnlyCommand import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -37,7 +36,6 @@ case class CreateIcebergView( allowExisting: Boolean, replace: Boolean, rewritten: Boolean = false, - materializedViewOptions: Option[MaterializedViewOptions] = None, isAnalyzed: Boolean = false) extends AnalysisOnlyCommand { diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala deleted file mode 100644 index c69417032709..000000000000 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.spark.sql.execution.datasources.v2 - -import org.apache.iceberg.catalog.Namespace -import org.apache.iceberg.catalog.TableIdentifier -import org.apache.iceberg.relocated.com.google.common.base.Preconditions -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap -import org.apache.iceberg.spark.MaterializedViewUtil -import org.apache.iceberg.spark.Spark3Util -import org.apache.iceberg.spark.SparkCatalog -import org.apache.iceberg.spark.SparkSchemaUtil -import org.apache.iceberg.spark.source.SparkView -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.connector.catalog.Identifier -import org.apache.spark.sql.connector.catalog.View -import org.apache.spark.sql.connector.catalog.ViewCatalog -import org.apache.spark.sql.connector.expressions.Transform -import org.apache.spark.sql.types.StructType -import scala.jdk.CollectionConverters._ - -case class CreateMaterializedViewExec( - catalog: ViewCatalog, - ident: Identifier, - queryText: String, - viewSchema: StructType, - columnAliases: Seq[String], - columnComments: Seq[Option[String]], - queryColumnNames: Seq[String], - comment: Option[String], - properties: Map[String, String], - allowExisting: Boolean, - replace: Boolean, - storageTableIdentifier: Option[String]) - extends LeafV2CommandExec { - - override def output: Seq[Attribute] = Nil - - override protected def run(): Seq[InternalRow] = { - - // Check if storageTableIdentifier is provided. If not, generate a default identifier. - val sparkStorageTableIdentifier = storageTableIdentifier match { - case Some(identifier) => { - val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(session, identifier) - val storageTableCatalogName = catalogAndIdentifier.catalog().name() - Preconditions.checkState( - storageTableCatalogName.equals(catalog.name()), - "Storage table identifier must be in the same catalog as the view." + - " Found storage table in catalog: %s, expected: %s.", - Array[Object](storageTableCatalogName, catalog.name())) - catalogAndIdentifier.identifier() - } - case None => MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier(ident) - } - - // Step 1: Create the storage table BEFORE the MV view metadata. - // Per spec: "The storage table must exist and be accessible before the - // materialized view metadata is committed." - // A newly created MV has a storage table with no snapshots until a refresh is performed. - catalog - .asInstanceOf[SparkCatalog] - .createTable( - sparkStorageTableIdentifier, - viewSchema, - new Array[Transform](0), - ImmutableMap.of[String, String]()) - - // Step 2: Create the MV view metadata with a storage-table reference - try { - createView(sparkStorageTableIdentifier.toString) match { - case Some(_) => // success - case None => // allowExisting and view already exists - } - } catch { - case e: Exception => - // If view creation fails, clean up the storage table - try { - catalog.asInstanceOf[SparkCatalog].dropTable(sparkStorageTableIdentifier) - } catch { - case _: Exception => // best effort cleanup - } - throw e - } - - Nil - } - - override def simpleString(maxFields: Int): String = { - s"CreateMaterializedViewExec: ${ident}" - } - - private def createView(storageTableIdentifier: String): Option[View] = { - val icebergSchema = SparkSchemaUtil.convert(viewSchema) - val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name - val currentCatalog = - if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null - val currentNamespace = session.sessionState.catalogManager.currentNamespace - - val engineVersion = "Spark " + org.apache.spark.SPARK_VERSION - val newProperties = properties ++ - comment.map(ViewCatalog.PROP_COMMENT -> _) + - ( - ViewCatalog.PROP_CREATE_ENGINE_VERSION -> engineVersion, - ViewCatalog.PROP_ENGINE_VERSION -> engineVersion) + - ("queryColumnNames" -> queryColumnNames.mkString(",")) - - if (replace) { - // CREATE OR REPLACE VIEW - if (catalog.viewExists(ident)) { - catalog.dropView(ident) - } - // FIXME: replaceView API doesn't exist in Spark 3.5 - val viewCatalog = catalog - .asInstanceOf[SparkCatalog] - .icebergCatalog() - .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val icebergView = viewCatalog - .buildView(Spark3Util.identifierToTableIdentifier(ident)) - .withDefaultCatalog(currentCatalog) - .withDefaultNamespace(Namespace.of(currentNamespace: _*)) - .withQuery("spark", queryText) - .withSchema(icebergSchema) - .withLocation(properties.get("location").orNull) - .withProperties(newProperties.asJava) - .withStorageTableIdentifier(TableIdentifier.parse(storageTableIdentifier)) - .create() - Some(new SparkView(catalog.name(), icebergView)) - - } else { - try { - // CREATE VIEW [IF NOT EXISTS] - val viewCatalog = catalog - .asInstanceOf[SparkCatalog] - .icebergCatalog() - .asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val icebergView = viewCatalog - .buildView(Spark3Util.identifierToTableIdentifier(ident)) - .withDefaultCatalog(currentCatalog) - .withDefaultNamespace(Namespace.of(currentNamespace: _*)) - .withQuery("spark", queryText) - .withSchema(icebergSchema) - .withLocation(properties.get("location").orNull) - .withProperties(newProperties.asJava) - .withStorageTableIdentifier(TableIdentifier.parse(storageTableIdentifier)) - .create() - Some(new SparkView(catalog.name(), icebergView)) - } catch { - // TODO: Make sure the existing view is also a materialized view - case _: ViewAlreadyExistsException if allowExisting => None - } - } - } - -} diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala index 84c111ae82ef..6dd1188b78e8 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala @@ -18,11 +18,6 @@ */ package org.apache.spark.sql.execution.datasources.v2 -import org.apache.iceberg.catalog.Namespace -import org.apache.iceberg.catalog.TableIdentifier -import org.apache.iceberg.exceptions -import org.apache.iceberg.spark.SparkCatalog -import org.apache.iceberg.view.View import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.NoSuchViewException import org.apache.spark.sql.catalyst.expressions.Attribute @@ -35,33 +30,6 @@ case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boo override lazy val output: Seq[Attribute] = Nil override protected def run(): Seq[InternalRow] = { - // If the catalog is a SparkCatalog, check for materialized view storage table cleanup - catalog match { - case sparkCatalog: SparkCatalog => - val icebergCatalog = sparkCatalog.icebergCatalog() - val icebergViewCatalog = icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - var view: Option[View] = None - try { - val ns = Namespace.of(ident.namespace(): _*) - val viewId = TableIdentifier.of(ns, ident.name()) - view = Some(icebergViewCatalog.loadView(viewId)) - } catch { - case _: exceptions.NoSuchViewException => - if (!ifExists) { - throw new NoSuchViewException(ident) - } - } - // if view is a materialized view, drop the storage table first - view.foreach { v => - val storageTable = v.currentVersion().storageTable() - if (storageTable != null) { - val storageIdent = Identifier.of(storageTable.namespace().levels(), storageTable.name()) - sparkCatalog.dropTable(storageIdent) - } - } - case _ => // not a SparkCatalog, skip MV cleanup - } - val dropped = catalog.dropView(ident) if (!dropped && !ifExists) { throw new NoSuchViewException(ident) diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala index 0a109e282a08..da540f5891b7 100644 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala +++ b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala @@ -36,7 +36,6 @@ import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField import org.apache.spark.sql.catalyst.plans.logical.DropTag import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.logical.OrderAwareCoalesce -import org.apache.spark.sql.catalyst.plans.logical.RefreshMaterializedViewStatement import org.apache.spark.sql.catalyst.plans.logical.RenameTable import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields @@ -142,35 +141,6 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi allowExisting, replace, _, - Some(materializedViewOptions), - _) => - CreateMaterializedViewExec( - catalog = viewCatalog, - ident = ident, - queryText = queryText, - columnAliases = columnAliases, - columnComments = columnComments, - queryColumnNames = queryColumnNames, - viewSchema = query.schema, - comment = comment, - properties = properties, - allowExisting = allowExisting, - replace = replace, - storageTableIdentifier = materializedViewOptions.storageTableIdentifier) :: Nil - - case CreateIcebergView( - ResolvedIdentifier(viewCatalog: ViewCatalog, ident), - queryText, - query, - columnAliases, - columnComments, - queryColumnNames, - comment, - properties, - allowExisting, - replace, - _, - None, _) => CreateV2ViewExec( catalog = viewCatalog, @@ -203,9 +173,6 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi case UnsetViewProperties(ResolvedV2View(catalog, ident), propertyKeys, ifExists) => AlterV2ViewUnsetPropertiesExec(catalog, ident, propertyKeys, ifExists) :: Nil - case RefreshMaterializedViewStatement(catalog, ident) => - RefreshMaterializedViewExec(catalog, ident) :: Nil - case _ => Nil } diff --git a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala b/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala deleted file mode 100644 index 055557319235..000000000000 --- a/spark/v4.1/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.spark.sql.execution.datasources.v2 - -import org.apache.iceberg.catalog.Namespace -import org.apache.iceberg.catalog.TableIdentifier -import org.apache.iceberg.relocated.com.google.common.base.Preconditions -import org.apache.iceberg.spark.SparkCatalog -import org.apache.iceberg.view.RefreshState -import org.apache.iceberg.view.RefreshStateParser -import org.apache.iceberg.view.SourceTableState -import org.apache.iceberg.view.SourceViewState -import org.apache.iceberg.view.SQLViewRepresentation -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.NoSuchTableException -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias -import org.apache.spark.sql.connector.catalog.CatalogManager -import org.apache.spark.sql.connector.catalog.Identifier -import org.apache.spark.sql.connector.catalog.LookupCatalog -import org.apache.spark.sql.connector.catalog.ViewCatalog -import org.apache.spark.sql.functions -import scala.jdk.CollectionConverters._ - -case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) - extends LeafV2CommandExec - with LookupCatalog { - - protected lazy val catalogManager: CatalogManager = session.sessionState.catalogManager - - override def output: Seq[Attribute] = Nil - - override protected def run(): Seq[InternalRow] = { - val sparkCatalog = catalog.asInstanceOf[SparkCatalog] - val icebergCatalog = sparkCatalog.icebergCatalog() - val icebergViewCatalog = - icebergCatalog.asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val viewId = TableIdentifier.of(Namespace.of(ident.namespace(): _*), ident.name()) - val view = icebergViewCatalog.loadView(viewId) - - val storageTableId = view.currentVersion().storageTable() - Preconditions.checkState( - storageTableId != null, - "Cannot refresh %s: not a materialized view (no storage table)", - ident) - - // Extract the SQL query from the view's representations - val sparkSql = view - .currentVersion() - .representations() - .asScala - .collect { case sql: SQLViewRepresentation if sql.dialect() == "spark" => sql.sql() } - .headOption - .getOrElse(throw new IllegalStateException( - s"Cannot refresh $ident: no Spark SQL representation found")) - - val refreshStartTimestampMs = System.currentTimeMillis() - - // Execute the view's query to get the current result set - val queryResult = session.sql(sparkSql) - - // Discover source tables and views from the query's logical plan and capture their - // current state - val sourceStates = collectSourceStates(queryResult.queryExecution.analyzed) - - // Build refresh state - val refreshState = new RefreshState( - view.currentVersion().versionId(), - sourceStates.asJava, - refreshStartTimestampMs) - val refreshStateJson = RefreshStateParser.toJson(refreshState) - - // Write results to storage table, replacing existing data - val storageTableRef = String.format( - "%s.%s.%s", - sparkCatalog.name(), - storageTableId.namespace().toString, - storageTableId.name()) - try { - queryResult - .writeTo(storageTableRef) - .option("snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) - .overwrite(functions.lit(true)) - } catch { - case e: NoSuchTableException => - throw new IllegalStateException( - s"Storage table $storageTableRef not found during refresh", - e) - } - - Nil - } - - private def collectSourceStates(plan: org.apache.spark.sql.catalyst.plans.logical.LogicalPlan) - : List[org.apache.iceberg.view.SourceState] = { - val sparkCatalog = catalog.asInstanceOf[SparkCatalog] - val icebergCatalog = - sparkCatalog.icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.Catalog] - val icebergViewCatalog = - sparkCatalog.icebergCatalog().asInstanceOf[org.apache.iceberg.catalog.ViewCatalog] - val seen = scala.collection.mutable.LinkedHashSet.empty[String] - val states = scala.collection.mutable.ListBuffer.empty[org.apache.iceberg.view.SourceState] - - plan.collectLeaves().foreach { - case r: org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation - if r.catalog.exists(_.name() == sparkCatalog.name()) => - val tableIdent = r.identifier.get - val key = "table:" + tableIdent.toString - if (seen.add(key)) { - val icebergId = - TableIdentifier.of(Namespace.of(tableIdent.namespace(): _*), tableIdent.name()) - try { - val table = icebergCatalog.loadTable(icebergId) - val snapshotId = - if (table.currentSnapshot() != null) { - table.currentSnapshot().snapshotId() - } else { - -1L - } - states += new SourceTableState( - icebergId.name(), - icebergId.namespace().levels().toList.asJava, - null, - table.uuid().toString, - snapshotId, - null) - } catch { - case _: Exception => // skip tables we can't load - } - } - case _ => // skip non-iceberg leaves - } - - // Spark's analyzer resolves every view reference into a SubqueryAlias wrapping the - // view's expanded query, including transitively for view-of-view chains, so a single - // pass over the whole plan (not just its leaves) discovers every source view at every - // nesting depth. - plan - .collect { case sub: SubqueryAlias => - sub.identifier.qualifier :+ sub.identifier.name - } - .collect { - case CatalogAndIdentifier(cat, viewIdent) if cat.name() == sparkCatalog.name() => viewIdent - } - .foreach { viewIdent => - val key = "view:" + viewIdent.toString - if (seen.add(key)) { - val icebergId = - TableIdentifier.of(Namespace.of(viewIdent.namespace(): _*), viewIdent.name()) - try { - val view = icebergViewCatalog.loadView(icebergId) - states += new SourceViewState( - icebergId.name(), - icebergId.namespace().levels().toList.asJava, - null, - view.uuid().toString, - view.currentVersion().versionId()) - } catch { - case _: Exception => // not a view, or the view can't be loaded - } - } - } - - states.toList - } - - override def simpleString(maxFields: Int): String = { - s"RefreshMaterializedViewExec: ${ident}" - } -} diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java deleted file mode 100644 index 167eb5d16cf5..000000000000 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ /dev/null @@ -1,495 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.spark.extensions; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.api.Assertions.fail; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.Arrays; -import java.util.Map; -import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.ParameterizedTestExtension; -import org.apache.iceberg.Parameters; -import org.apache.iceberg.TableOperations; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.RuntimeIOException; -import org.apache.iceberg.inmemory.InMemoryCatalog; -import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.io.OutputFile; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; -import org.apache.iceberg.spark.MaterializedViewUtil; -import org.apache.iceberg.spark.SparkCatalog; -import org.apache.iceberg.spark.SparkCatalogConfig; -import org.apache.iceberg.spark.source.SparkMaterializedView; -import org.apache.iceberg.spark.source.SparkView; -import org.apache.iceberg.view.RefreshState; -import org.apache.iceberg.view.RefreshStateParser; -import org.apache.iceberg.view.SourceTableState; -import org.apache.iceberg.view.SourceViewState; -import org.apache.iceberg.view.View; -import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; -import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; -import org.apache.spark.sql.connector.catalog.CatalogPlugin; -import org.apache.spark.sql.connector.catalog.Identifier; -import org.apache.spark.sql.connector.catalog.TableCatalog; -import org.apache.spark.sql.connector.catalog.ViewCatalog; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.api.extension.ExtendWith; - -@ExtendWith(ParameterizedTestExtension.class) -public class TestMaterializedViews extends ExtensionsTestBase { - private static final Namespace NAMESPACE = Namespace.of("default"); - private final String tableName = "table"; - private final String materializedViewName = "materialized_view"; - - @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") - protected static Object[][] parameters() { - Map properties = - Maps.newHashMap(SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.properties()); - properties.put(CatalogProperties.WAREHOUSE_LOCATION, "file:" + getTempWarehouseDir()); - properties.put(CatalogProperties.CATALOG_IMPL, InMemoryCatalogWithLocalFileIO.class.getName()); - return new Object[][] { - { - SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.catalogName(), - SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.implementation(), - properties - } - }; - } - - private static String getTempWarehouseDir() { - try { - File tempDir = Files.createTempDirectory("warehouse-").toFile(); - tempDir.deleteOnExit(); - return tempDir.getAbsolutePath(); - - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @BeforeEach - @Override - public void before() { - // Set up a simple InMemoryCatalog as validation catalog to avoid base class - // configureValidationCatalog() failing on our custom catalog-impl. - this.validationCatalog = new InMemoryCatalog(); - this.validationNamespaceCatalog = - (org.apache.iceberg.catalog.SupportsNamespaces) validationCatalog; - - spark.conf().set("spark.sql.catalog." + catalogName, implementation); - catalogConfig.forEach( - (key, value) -> spark.conf().set("spark.sql.catalog." + catalogName + "." + key, value)); - - sql("CREATE NAMESPACE IF NOT EXISTS default"); - spark.conf().set("spark.sql.defaultCatalog", catalogName); - sql("USE %s", catalogName); - sql("CREATE NAMESPACE IF NOT EXISTS %s", NAMESPACE); - sql("CREATE TABLE %s (id INT, data STRING)", tableName); - } - - @AfterEach - public void removeTable() { - sql("USE %s", catalogName); - sql("DROP VIEW IF EXISTS %s", materializedViewName); - sql("DROP TABLE IF EXISTS %s", tableName); - } - - @TestTemplate - public void testStorageTableFieldOnViewVersion() { - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - View view = loadIcebergView(); - // storage-table should be set on the view version, not as a property - assertThat(view.currentVersion().storageTable()).isNotNull(); - assertThat(view.currentVersion().storageTable().name()) - .isEqualTo(materializedViewName + "__storage"); - assertThat(view.currentVersion().storageTable().namespace()).isEqualTo(NAMESPACE); - } - - @TestTemplate - public void testNeverRefreshedMvIsNotFresh() { - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - // A newly created MV has no snapshots on its storage table, so it's not fresh. - // loadView should succeed (returns stale view) - try { - assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); - } catch (NoSuchViewException e) { - fail("Materialized view not found"); - } - } - - @TestTemplate - public void testReadFromStorageTableWhenFresh() { - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - simulateRefresh(); - - // Fresh MV: loadTable should return SparkMaterializedView - try { - assertThat(sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(SparkMaterializedView.class); - } catch (NoSuchTableException e) { - fail("Fresh materialized view should be loadable as a table"); - } - - // Fresh MV: loadView should throw since the engine should use loadTable instead - assertThatThrownBy(() -> sparkViewCatalog().loadView(viewIdentifier())) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("fresh"); - } - - @TestTemplate - public void testFallbackToViewWhenStale() { - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - simulateRefresh(); - - // Insert more data to invalidate the refresh - sql("INSERT INTO %s VALUES (3, 'c')", tableName); - - // Stale MV: loadView should return SparkView (falls back to query execution) - try { - assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); - } catch (NoSuchViewException e) { - fail("Stale materialized view should be loadable as a view"); - } - - // Stale MV: loadTable should not resolve to the MV's storage table - assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(NoSuchTableException.class) - .hasMessageContaining(materializedViewName); - } - - @TestTemplate - public void testStorageTableCreatedBeforeMvMetadata() { - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - // The storage table should exist - String storageTableName = - MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier( - Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName)) - .name(); - assertThat(sql("SHOW TABLES")) - .anySatisfy(row -> assertThat(row[1]).isEqualTo(storageTableName)); - } - - @TestTemplate - public void testDefaultStorageTableNaming() { - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - // Default naming should be __storage - String expectedStorageTableName = materializedViewName + "__storage"; - assertThat(sql("SHOW TABLES")) - .anySatisfy(row -> assertThat(row[1]).isEqualTo(expectedStorageTableName)); - } - - @TestTemplate - public void testStoredAsClause() { - String customTableName = "custom_table_name"; - sql( - "CREATE MATERIALIZED VIEW %s STORED AS '%s' AS SELECT id, data FROM %s", - materializedViewName, customTableName, tableName); - - // Assert that the storage table with the custom name is in the list of tables - assertThat(sql("SHOW TABLES")).anySatisfy(row -> assertThat(row[1]).isEqualTo(customTableName)); - } - - @TestTemplate - public void testRefreshMaterializedView() { - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - // Refresh the materialized view - sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); - - // After refresh, the MV should be fresh and loadable as a table - try { - assertThat(sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(SparkMaterializedView.class); - } catch (NoSuchTableException e) { - fail("Refreshed materialized view should be loadable as a table"); - } - - // Verify the storage table has data - View view = loadIcebergView(); - String storageTableRef = - String.format( - "%s.%s.%s", catalogName, NAMESPACE, view.currentVersion().storageTable().name()); - assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(2); - } - - @TestTemplate - public void testRefreshMaterializedViewUpdatesData() { - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); - sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); - - // First refresh - sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); - - // Insert more data - sql("INSERT INTO %s VALUES (3, 'c')", tableName); - - // Before second refresh, the MV should be stale - try { - assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); - } catch (NoSuchViewException e) { - fail("Stale materialized view should be loadable as a view"); - } - - // Second refresh - sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); - - // After refresh, the MV should be fresh again - try { - assertThat(sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(SparkMaterializedView.class); - } catch (NoSuchTableException e) { - fail("Refreshed materialized view should be loadable as a table"); - } - - // Verify the storage table has all 3 rows - View view = loadIcebergView(); - String storageTableRef = - String.format( - "%s.%s.%s", catalogName, NAMESPACE, view.currentVersion().storageTable().name()); - assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(3); - } - - @TestTemplate - public void testRefreshRecordsNestedViewState() { - String sourceViewName = "source_view"; - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); - sql("CREATE VIEW %s AS SELECT id, data FROM %s", sourceViewName, tableName); - sql( - "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", - materializedViewName, sourceViewName); - - sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); - - View sourceView = loadIcebergView(sourceViewName); - RefreshState refreshState = loadRefreshState(); - - // The refresh state should record both the nested source view and the base table it - // resolves to, since the analyzed query plan fully expands the view chain. - assertThat(refreshState.sourceStates()).hasSize(2); - - SourceViewState viewState = - refreshState.sourceStates().stream() - .filter(SourceViewState.class::isInstance) - .map(SourceViewState.class::cast) - .findFirst() - .orElseGet(() -> fail("Refresh state should record the nested source view")); - assertThat(viewState.name()).isEqualTo(sourceViewName); - assertThat(viewState.namespace()).isEqualTo(Arrays.asList(NAMESPACE.levels())); - assertThat(viewState.uuid()).isEqualTo(sourceView.uuid().toString()); - assertThat(viewState.versionId()).isEqualTo(sourceView.currentVersion().versionId()); - - SourceTableState tableState = - refreshState.sourceStates().stream() - .filter(SourceTableState.class::isInstance) - .map(SourceTableState.class::cast) - .findFirst() - .orElseGet(() -> fail("Refresh state should record the underlying base table")); - assertThat(tableState.name()).isEqualTo(tableName); - - sql("DROP VIEW IF EXISTS %s", sourceViewName); - } - - @TestTemplate - public void testStaleWhenNestedViewChanges() { - String sourceViewName = "source_view"; - sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); - sql("CREATE VIEW %s AS SELECT id, data FROM %s WHERE id <= 2", sourceViewName, tableName); - sql( - "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", - materializedViewName, sourceViewName); - - sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); - - // Freshly refreshed: loadable as a table - try { - assertThat(sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(SparkMaterializedView.class); - } catch (NoSuchTableException e) { - fail("Refreshed materialized view should be loadable as a table"); - } - - // Replace the nested view's definition without touching the base table. This bumps the - // source view's version but leaves the underlying base table's snapshot unchanged. - sql( - "CREATE OR REPLACE VIEW %s AS SELECT id, data FROM %s WHERE id <= 1", - sourceViewName, tableName); - - // The MV should now be stale because its nested source view changed versions, even - // though the underlying base table's snapshot did not change. - try { - assertThat(sparkViewCatalog().loadView(viewIdentifier())).isInstanceOf(SparkView.class); - } catch (NoSuchViewException e) { - fail("Materialized view with a stale nested view should be loadable as a view"); - } - assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) - .isInstanceOf(NoSuchTableException.class) - .hasMessageContaining(materializedViewName); - - sql("DROP VIEW IF EXISTS %s", sourceViewName); - } - - private RefreshState loadRefreshState() { - View view = loadIcebergView(); - org.apache.iceberg.catalog.TableIdentifier storageTableId = - view.currentVersion().storageTable(); - org.apache.iceberg.Table storageTable = - sparkCatalog().icebergCatalog().loadTable(storageTableId); - String refreshStateJson = - storageTable.currentSnapshot().summary().get(RefreshState.REFRESH_STATE_SUMMARY_KEY); - return RefreshStateParser.fromJson(refreshStateJson); - } - - private void simulateRefresh() { - View view = loadIcebergView(); - org.apache.iceberg.catalog.TableIdentifier storageTableId = - view.currentVersion().storageTable(); - - // Get the base table's current snapshot ID - long baseSnapshotId = - (Long) - sql( - "SELECT snapshot_id FROM %s.%s.%s.snapshots ORDER BY committed_at DESC LIMIT 1", - catalogName, NAMESPACE, tableName) - .get(0)[0]; - - // Build refresh state matching the current view version and source table state - RefreshState refreshState = - new RefreshState( - view.currentVersion().versionId(), - Arrays.asList( - new SourceTableState( - tableName, - Arrays.asList(NAMESPACE.levels()), - null, - "test-uuid", - baseSnapshotId, - null)), - System.currentTimeMillis()); - String refreshStateJson = RefreshStateParser.toJson(refreshState); - - // Write data to storage table with refresh-state in the snapshot summary - String storageTableRef = - String.format("%s.%s.%s", catalogName, NAMESPACE, storageTableId.name()); - try { - spark - .sql(String.format("SELECT id, data FROM %s.%s.%s", catalogName, NAMESPACE, tableName)) - .writeTo(storageTableRef) - .option("snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) - .append(); - } catch (NoSuchTableException e) { - throw new RuntimeException("Storage table not found during simulated refresh", e); - } - } - - private ViewCatalog sparkViewCatalog() { - CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); - return (ViewCatalog) catalogPlugin; - } - - private TableCatalog sparkTableCatalog() { - CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); - return (TableCatalog) catalogPlugin; - } - - private Identifier viewIdentifier() { - return Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName); - } - - private SparkCatalog sparkCatalog() { - return (SparkCatalog) spark.sessionState().catalogManager().catalog(catalogName); - } - - private View loadIcebergView() { - return loadIcebergView(materializedViewName); - } - - private View loadIcebergView(String viewName) { - org.apache.iceberg.catalog.ViewCatalog icebergViewCatalog = - (org.apache.iceberg.catalog.ViewCatalog) sparkCatalog().icebergCatalog(); - return icebergViewCatalog.loadView(TableIdentifier.of(NAMESPACE, viewName)); - } - - // Required to be public since it is loaded by org.apache.iceberg.CatalogUtil.loadCatalog - public static class InMemoryCatalogWithLocalFileIO extends InMemoryCatalog { - private FileIO localFileIO; - - @Override - public void initialize(String name, Map properties) { - super.initialize(name, properties); - localFileIO = new LocalFileIO(); - } - - @Override - protected TableOperations newTableOps(TableIdentifier tableIdentifier) { - return new InMemoryTableOperations(localFileIO, tableIdentifier); - } - - @Override - protected InMemoryCatalog.InMemoryViewOperations newViewOps(TableIdentifier identifier) { - return new InMemoryViewOperations(localFileIO, identifier); - } - } - - private static class LocalFileIO implements FileIO { - - private static String stripFilePrefix(String path) { - return path.startsWith("file:") ? path.substring(5) : path; - } - - @Override - public InputFile newInputFile(String path) { - return org.apache.iceberg.Files.localInput(stripFilePrefix(path)); - } - - @Override - public OutputFile newOutputFile(String path) { - String stripped = stripFilePrefix(path); - java.io.File parent = new java.io.File(stripped).getParentFile(); - if (!parent.isDirectory()) { - parent.mkdirs(); - } - return org.apache.iceberg.Files.localOutput(stripped); - } - - @Override - public void deleteFile(String path) { - if (!new File(stripFilePrefix(path)).delete()) { - throw new RuntimeIOException("Failed to delete file: " + path); - } - } - } -} diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java deleted file mode 100644 index a30c5f671176..000000000000 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.spark; - -import org.apache.spark.sql.connector.catalog.Identifier; - -public class MaterializedViewUtil { - - private MaterializedViewUtil() {} - - private static final String MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX = "__storage"; - - public static Identifier getDefaultMaterializedViewStorageTableIdentifier( - Identifier viewIdentifier) { - return Identifier.of( - viewIdentifier.namespace(), - viewIdentifier.name() + MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX); - } -} diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index 19b57c094abb..40db152076c8 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -58,7 +58,6 @@ import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.spark.actions.SparkActions; import org.apache.iceberg.spark.source.SparkChangelogTable; -import org.apache.iceberg.spark.source.SparkMaterializedView; import org.apache.iceberg.spark.source.SparkTable; import org.apache.iceberg.spark.source.SparkView; import org.apache.iceberg.spark.source.StagedSparkTable; @@ -599,12 +598,7 @@ public View loadView(Identifier ident) throws NoSuchViewException { if (null != asViewCatalog) { try { org.apache.iceberg.view.View view = asViewCatalog.loadView(buildIdentifier(ident)); - if (isMaterializedView(view) && isFresh(view)) { - throw new IllegalStateException( - "Materialized view is fresh. loadTable should be attempted instead."); - } else { - return new SparkView(catalogName, view); - } + return new SparkView(catalogName, view); } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { throw new NoSuchViewException(ident); } @@ -613,95 +607,6 @@ public View loadView(Identifier ident) throws NoSuchViewException { throw new NoSuchViewException(ident); } - private boolean isMaterializedView(org.apache.iceberg.view.View view) { - return view.currentVersion().storageTable() != null; - } - - private org.apache.iceberg.catalog.TableIdentifier getStorageTableId( - org.apache.iceberg.view.View view) { - org.apache.iceberg.catalog.TableIdentifier storageTable = view.currentVersion().storageTable(); - Preconditions.checkState( - storageTable != null, "Storage table identifier is not set for materialized view."); - return storageTable; - } - - private Table loadStorageTable(org.apache.iceberg.view.View view) { - org.apache.iceberg.catalog.TableIdentifier storageTableId = getStorageTableId(view); - try { - Identifier sparkIdent = - Identifier.of(storageTableId.namespace().levels(), storageTableId.name()); - return loadTable(sparkIdent); - } catch (NoSuchTableException e) { - throw new IllegalStateException("Unable to load storage table for materialized view.", e); - } - } - - private boolean isFresh(org.apache.iceberg.view.View view) { - Table sparkStorageTable = loadStorageTable(view); - org.apache.iceberg.Table storageTable = ((SparkTable) sparkStorageTable).table(); - if (storageTable.currentSnapshot() == null) { - return false; - } - - String refreshStateJson = - storageTable - .currentSnapshot() - .summary() - .get(org.apache.iceberg.view.RefreshState.REFRESH_STATE_SUMMARY_KEY); - if (refreshStateJson == null) { - return false; - } - - org.apache.iceberg.view.RefreshState refreshState = - org.apache.iceberg.view.RefreshStateParser.fromJson(refreshStateJson); - - if (refreshState.viewVersionId() != view.currentVersion().versionId()) { - return false; - } - - for (org.apache.iceberg.view.SourceState sourceState : refreshState.sourceStates()) { - if (sourceState instanceof org.apache.iceberg.view.SourceTableState) { - org.apache.iceberg.view.SourceTableState tableState = - (org.apache.iceberg.view.SourceTableState) sourceState; - org.apache.iceberg.catalog.TableIdentifier sourceId = - org.apache.iceberg.catalog.TableIdentifier.of( - org.apache.iceberg.catalog.Namespace.of( - tableState.namespace().toArray(new String[0])), - tableState.name()); - try { - org.apache.iceberg.Table sourceTable = icebergCatalog().loadTable(sourceId); - long currentSnapshotId = - sourceTable.currentSnapshot() == null - ? -1 - : sourceTable.currentSnapshot().snapshotId(); - if (currentSnapshotId != tableState.snapshotId()) { - return false; - } - } catch (Exception e) { - return false; - } - } else if (sourceState instanceof org.apache.iceberg.view.SourceViewState) { - org.apache.iceberg.view.SourceViewState viewState = - (org.apache.iceberg.view.SourceViewState) sourceState; - org.apache.iceberg.catalog.TableIdentifier sourceId = - org.apache.iceberg.catalog.TableIdentifier.of( - org.apache.iceberg.catalog.Namespace.of( - viewState.namespace().toArray(new String[0])), - viewState.name()); - try { - org.apache.iceberg.view.View sourceView = asViewCatalog.loadView(sourceId); - if (sourceView.currentVersion().versionId() != viewState.versionId()) { - return false; - } - } catch (Exception e) { - return false; - } - } - } - - return true; - } - @Override public View createView(ViewInfo viewInfo) throws ViewAlreadyExistsException, NoSuchNamespaceException { @@ -992,19 +897,6 @@ private Table load(Identifier ident, TimeTravel timeTravel) throws NoSuchTableEx return loadPath((PathIdentifier) ident, timeTravel); } - // Check if materialized view. If fresh, return the SparkMaterializedView. - if (null != asViewCatalog) { - try { - org.apache.iceberg.view.View view = asViewCatalog.loadView(buildIdentifier(ident)); - if (isMaterializedView(view) && isFresh(view)) { - Table storageTable = loadStorageTable(view); - return new SparkMaterializedView(catalogName, view, storageTable); - } - } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { - // Ignore. Just process as a normal table. - } - } - try { org.apache.iceberg.Table table = icebergCatalog.loadTable(buildIdentifier(ident)); return SparkTable.create(table, timeTravel); diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java deleted file mode 100644 index 0c01ae449292..000000000000 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.spark.source; - -import java.util.Set; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; -import org.apache.iceberg.view.View; -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.connector.catalog.SupportsRead; -import org.apache.spark.sql.connector.catalog.Table; -import org.apache.spark.sql.connector.catalog.TableCapability; -import org.apache.spark.sql.connector.read.ScanBuilder; -import org.apache.spark.sql.util.CaseInsensitiveStringMap; - -public class SparkMaterializedView extends SparkView implements SupportsRead { - private final Table storageTable; - private SparkSession lazySpark; - - public SparkMaterializedView(String catalogName, View icebergView, Table storageTable) { - super(catalogName, icebergView); - this.storageTable = storageTable; - } - - @Override - public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) { - return ((SupportsRead) storageTable).newScanBuilder(options); - } - - private SparkSession sparkSession() { - if (lazySpark == null) { - this.lazySpark = SparkSession.active(); - } - - return lazySpark; - } - - @Override - public Set capabilities() { - return ImmutableSet.of(TableCapability.BATCH_READ); - } -} diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java index d02353a85b9b..b20c87619ed8 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java @@ -81,17 +81,7 @@ public enum SparkCatalogConfig { ImmutableMap.of( "type", "hive", "default-namespace", "default", - "unique-table-location", "true")), - SPARK_WITH_MATERIALIZED_VIEWS( - "spark_with_mvs", - SparkCatalog.class.getName(), - ImmutableMap.of( - CatalogProperties.CATALOG_IMPL, - InMemoryCatalog.class.getName(), - "default-namespace", - "default", - "cache-enabled", - "false")); + "unique-table-location", "true")); private final String catalogName; private final String implementation; From 4df4eb535bf86d49038e8b212d728ccdb7701a6e Mon Sep 17 00:00:00 2001 From: wmoustafa Date: Mon, 7 Sep 2026 22:34:46 -0700 Subject: [PATCH 21/22] Spark 4.2: Support materialized views Add CREATE MATERIALIZED VIEW, REFRESH MATERIALIZED VIEW and DROP for materialized views, following the view spec's materialized view design: a materialized view is a view whose current version names a storage table, and the storage table's snapshot summary records the state of the sources the result was computed from. Reading a materialized view goes through RelationCatalog.loadRelation, which compares the recorded source state against the sources as they are now. When they agree the storage table is read, and when they do not the view's query is read instead, so a stale materialized view returns the same rows as the view it materializes rather than stale ones. A refresh runs the view's query and writes the result to the storage table. Each view column takes its values from the query column recorded for it when the view was created, matching how Spark reads the view, so column aliases and a reordered source table both keep returning the same values. When a recorded name is no longer in the query's output the refresh fails and names the columns it cannot read. Replacing a materialized view is rejected. The spec records the storage table per view version and leaves open what a new version does with the table the previous definition materialized, so the statement is rejected rather than settling that here. CREATE OR REPLACE VIEW over a materialized view is rejected for the same reason. --- .../org/apache/iceberg/view/RefreshState.java | 3 + .../analysis/RewriteViewCommands.scala | 11 +- .../IcebergSparkSqlExtensionsParser.scala | 44 + .../RefreshMaterializedViewStatement.scala | 28 + .../logical/views/CreateIcebergView.scala | 2 + .../v2/CreateMaterializedViewExec.scala | 182 +++ .../datasources/v2/DropV2ViewExec.scala | 31 + .../v2/ExtendedDataSourceV2Strategy.scala | 34 + .../v2/RefreshMaterializedViewExec.scala | 252 ++++ .../extensions/TestMaterializedViews.java | 1040 +++++++++++++++++ .../iceberg/spark/MaterializedViewUtil.java | 35 + .../apache/iceberg/spark/SparkCatalog.java | 218 +++- .../spark/source/SparkMaterializedView.java | 80 ++ .../iceberg/spark/SparkCatalogConfig.java | 10 + 14 files changed, 1963 insertions(+), 7 deletions(-) create mode 100644 spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala create mode 100644 spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala create mode 100644 spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala create mode 100644 spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java create mode 100644 spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java create mode 100644 spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java diff --git a/core/src/main/java/org/apache/iceberg/view/RefreshState.java b/core/src/main/java/org/apache/iceberg/view/RefreshState.java index 160f280ac79c..892d27de1818 100644 --- a/core/src/main/java/org/apache/iceberg/view/RefreshState.java +++ b/core/src/main/java/org/apache/iceberg/view/RefreshState.java @@ -29,6 +29,9 @@ public class RefreshState { public static final String REFRESH_STATE_SUMMARY_KEY = "refresh-state"; + /** Recorded as the snapshot id of a source table that had no snapshot when it was read. */ + public static final long NO_SNAPSHOT_ID = -1L; + private final int viewVersionId; private final List sourceStates; private final long refreshStartTimestampMs; diff --git a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala index 3cd1cba0d66d..cfd100a3743f 100644 --- a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala +++ b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RewriteViewCommands.scala @@ -40,7 +40,11 @@ import org.apache.spark.sql.connector.catalog.LookupCatalog * ResolveSessionCatalog exits early for some v2 View commands, * thus they are pre-substituted here before Spark routes them through the V1 path. */ -case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] with LookupCatalog { +case class RewriteViewCommands( + spark: SparkSession, + materializedViewOptions: Option[MaterializedViewOptions] = None) + extends Rule[LogicalPlan] + with LookupCatalog { protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager @@ -85,7 +89,8 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi properties = properties, allowExisting = allowExisting, replace = replace, - viewSchemaMode = viewSchemaMode) + viewSchemaMode = viewSchemaMode, + materializedViewOptions = materializedViewOptions) case view @ ShowViews(CurrentNamespace, pattern, output) => if (ViewUtil.isIcebergViewCatalog(catalogManager.currentCatalog)) { @@ -158,3 +163,5 @@ case class RewriteViewCommands(spark: SparkSession) extends Rule[LogicalPlan] wi } } + +case class MaterializedViewOptions(storageTableIdentifier: Option[String]) diff --git a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala index 7c737f0513ed..72587bf2138d 100644 --- a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala +++ b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.MaterializedViewOptions import org.apache.spark.sql.catalyst.analysis.RewriteViewCommands import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.ParameterContext @@ -38,7 +39,9 @@ import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.NonReservedContext import org.apache.spark.sql.catalyst.parser.extensions.IcebergSqlExtensionsParser.QuotedIdentifierContext import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.plans.logical.RefreshMaterializedViewStatement import org.apache.spark.sql.catalyst.trees.Origin +import org.apache.spark.sql.connector.catalog.ViewCatalog import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.VariableSubstitution import org.apache.spark.sql.types.DataType @@ -53,6 +56,9 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) private lazy val substitutor = substitutorCtor.newInstance(SQLConf.get) private lazy val astBuilder = new IcebergSqlExtensionsAstBuilder(delegate) + private lazy final val CREATE_MATERIALIZED_VIEW_PATTERN = + "(?i)(CREATE(?:\\s+OR\\s+REPLACE)?)\\s+MATERIALIZED\\s+(VIEW)".r + private lazy final val MATERIALIZED_VIEW_STORED_AS_PATTERN = "(?i)STORED AS\\s*'(\\w+)'\\s*".r /** * Parse a string to a DataType. @@ -142,11 +148,49 @@ class IcebergSparkSqlExtensionsParser(delegate: ParserInterface) if (isIcebergCommand(sqlTextAfterSubstitution)) { parse(sqlTextAfterSubstitution) { parser => astBuilder.visit(parser.singleStatement()) } .asInstanceOf[LogicalPlan] + } else if (isCreateMaterializedView(sqlText)) { + RewriteViewCommands(SparkSession.active, Option(getMaterializedViewOptions(sqlText))) + .apply(delegate.parsePlan(getCreateMaterializedViewStatement(sqlText))) + } else if (isRefreshMaterializedView(sqlText)) { + parseRefreshMaterializedView(sqlText) } else { RewriteViewCommands(SparkSession.active).apply(delegateParse(sqlText)) } } + private def isCreateMaterializedView(sqlText: String): Boolean = { + CREATE_MATERIALIZED_VIEW_PATTERN.findFirstIn(sqlText).isDefined + } + + private def getCreateMaterializedViewStatement(sqlText: String): String = { + val createViewSql = + CREATE_MATERIALIZED_VIEW_PATTERN.replaceAllIn(sqlText, m => m.group(1) + " " + m.group(2)) + MATERIALIZED_VIEW_STORED_AS_PATTERN.replaceAllIn(createViewSql, "") + } + + private def getMaterializedViewOptions(sqlText: String): MaterializedViewOptions = { + val storedAsPattern = "(?i)STORED AS\\s*'(\\w+)'\\s*".r + val storageTableIdentifier = storedAsPattern.findFirstMatchIn(sqlText).map(_.group(1)) + MaterializedViewOptions(storageTableIdentifier) + } + + private def isRefreshMaterializedView(sqlText: String): Boolean = { + sqlText.toLowerCase.trim.startsWith("refresh materialized view") + } + + private def parseRefreshMaterializedView(sqlText: String): LogicalPlan = { + val viewName = sqlText.trim + .replaceFirst("(?i)REFRESH\\s+MATERIALIZED\\s+VIEW\\s+", "") + .trim + val spark = SparkSession.active + val catalogAndIdent = + org.apache.iceberg.spark.Spark3Util.catalogAndIdentifier(spark, viewName) + val viewCatalog = + catalogAndIdent.catalog().asInstanceOf[ViewCatalog] + val ident = catalogAndIdent.identifier() + RefreshMaterializedViewStatement(viewCatalog, ident) + } + private def isIcebergCommand(sqlText: String): Boolean = { val normalized = sqlText .toLowerCase(Locale.ROOT) diff --git a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala new file mode 100644 index 000000000000..8de8f2deaa4c --- /dev/null +++ b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/RefreshMaterializedViewStatement.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.catalyst.plans.logical + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.ViewCatalog + +case class RefreshMaterializedViewStatement(catalog: ViewCatalog, ident: Identifier) + extends LeafCommand { + override def output: Seq[Attribute] = Nil +} diff --git a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala index b976bfa6fef0..b013596bffb2 100644 --- a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala +++ b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/views/CreateIcebergView.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.plans.logical.views import org.apache.spark.sql.catalyst.analysis.AnalysisContext +import org.apache.spark.sql.catalyst.analysis.MaterializedViewOptions import org.apache.spark.sql.catalyst.analysis.ViewSchemaMode import org.apache.spark.sql.catalyst.plans.logical.AnalysisOnlyCommand import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -36,6 +37,7 @@ case class CreateIcebergView( allowExisting: Boolean, replace: Boolean, viewSchemaMode: ViewSchemaMode, + materializedViewOptions: Option[MaterializedViewOptions] = None, isAnalyzed: Boolean = false, referredTempFunctions: Seq[String] = Seq.empty) extends AnalysisOnlyCommand { diff --git a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala new file mode 100644 index 000000000000..0aa7bdf573c4 --- /dev/null +++ b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CreateMaterializedViewExec.scala @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.v2 + +import org.apache.iceberg.catalog.Namespace +import org.apache.iceberg.catalog.TableIdentifier +import org.apache.iceberg.relocated.com.google.common.base.Preconditions +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap +import org.apache.iceberg.spark.MaterializedViewUtil +import org.apache.iceberg.spark.Spark3Util +import org.apache.iceberg.spark.SparkCatalog +import org.apache.iceberg.spark.SparkSchemaUtil +import org.apache.iceberg.spark.source.SparkView +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException +import org.apache.spark.sql.catalyst.analysis.ViewUtil +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.TableCatalog +import org.apache.spark.sql.connector.catalog.View +import org.apache.spark.sql.connector.catalog.ViewCatalog +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.types.StructType +import scala.jdk.CollectionConverters._ + +case class CreateMaterializedViewExec( + catalog: ViewCatalog, + ident: Identifier, + queryText: String, + viewSchema: StructType, + columnAliases: Seq[String], + columnComments: Seq[Option[String]], + queryColumnNames: Seq[String], + comment: Option[String], + properties: Map[String, String], + allowExisting: Boolean, + replace: Boolean, + storageTableIdentifier: Option[String]) + extends LeafV2CommandExec { + + override def output: Seq[Attribute] = Nil + + /** + * The columns of the materialized view, which are the columns of the query renamed by the column + * aliases the statement declared. The query column names are recorded separately, so the aliases + * name the view while the query keeps its own output names. + */ + private lazy val outputSchema: StructType = { + if (columnAliases.isEmpty) { + viewSchema + } else { + StructType(viewSchema.fields.zipWithIndex.map { case (field, i) => + val renamed = field.copy(name = columnAliases(i)) + columnComments(i).map(renamed.withComment).getOrElse(renamed) + }) + } + } + + override protected def run(): Seq[InternalRow] = { + // Replacing a materialized view has to decide what becomes of the storage table that the + // previous definition materialized, and the view spec leaves that open: the storage table + // identifier is recorded per view version, so a new version may keep the existing table or + // point at a different one. Rather than settle that here, the statement is rejected. Drop + // the materialized view and create it again to change its definition. + if (replace) { + throw new UnsupportedOperationException( + s"Cannot replace materialized view: $ident. " + + "Drop the materialized view and create it again to change its definition") + } + + // Check if storageTableIdentifier is provided. If not, generate a default identifier. + val sparkStorageTableIdentifier = storageTableIdentifier match { + case Some(identifier) => { + val catalogAndIdentifier = Spark3Util.catalogAndIdentifier(session, identifier) + val storageTableCatalogName = catalogAndIdentifier.catalog().name() + Preconditions.checkState( + storageTableCatalogName.equals(catalog.name()), + "Storage table identifier must be in the same catalog as the view." + + " Found storage table in catalog: %s, expected: %s.", + Array[Object](storageTableCatalogName, catalog.name())) + catalogAndIdentifier.identifier() + } + case None => MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier(ident) + } + + // Step 1: Create the storage table BEFORE the MV view metadata. + // Per spec: "The storage table must exist and be accessible before the + // materialized view metadata is committed." + // A newly created MV has a storage table with no snapshots until a refresh is performed. + val sparkCatalog = catalog.asInstanceOf[SparkCatalog] + sparkCatalog + .createTable( + sparkStorageTableIdentifier, + outputSchema, + new Array[Transform](0), + ImmutableMap.of[String, String]()) + + // Step 2: Create the MV view metadata with a storage-table reference + try { + createView(sparkStorageTableIdentifier.toString) match { + case Some(_) => // success + case None => // allowExisting and view already exists + } + } catch { + case e: Exception => + try { + sparkCatalog.dropTable(sparkStorageTableIdentifier) + } catch { + case _: Exception => // best effort cleanup + } + + throw e + } + + Nil + } + + override def simpleString(maxFields: Int): String = { + s"CreateMaterializedViewExec: ${ident}" + } + + private def createView(storageTableIdentifier: String): Option[View] = { + val icebergSchema = SparkSchemaUtil.convert(outputSchema) + val currentCatalogName = session.sessionState.catalogManager.currentCatalog.name + val currentCatalog = + if (!catalog.name().equals(currentCatalogName)) currentCatalogName else null + val currentNamespace = session.sessionState.catalogManager.currentNamespace + + // The reserved properties that carry Spark view metadata are composed the same way as for a + // plain view, so that a materialized view and a view are described by the same property keys. + // Among them are the query's column names, which record the query column each view column + // takes its values from, and which a refresh reads back to pair them up again by name. + val sparkView = new View.Builder() + .withQueryText(queryText) + .withCurrentCatalog(currentCatalog) + .withCurrentNamespace(currentNamespace) + .withSchema(outputSchema) + .withQueryColumnNames(queryColumnNames.toArray) + .withSqlConfigs(ImmutableMap.of[String, String]()) + .withProperties((properties ++ comment.map(TableCatalog.PROP_COMMENT -> _)).asJava) + .build() + val newProperties = ViewUtil.createProperties(sparkView).asScala.toMap + + try { + // CREATE VIEW [IF NOT EXISTS] + val viewCatalog = catalog + .asInstanceOf[SparkCatalog] + .icebergViewCatalog() + val icebergView = viewCatalog + .buildView(Spark3Util.identifierToTableIdentifier(ident)) + .withDefaultCatalog(currentCatalog) + .withDefaultNamespace(Namespace.of(currentNamespace: _*)) + .withQuery("spark", queryText) + .withSchema(icebergSchema) + .withLocation(properties.get("location").orNull) + .withProperties(newProperties.asJava) + .withStorageTableIdentifier(TableIdentifier.parse(storageTableIdentifier)) + .create() + Some(SparkView.toView(catalog.name(), icebergView)) + } catch { + // TODO: Make sure the existing view is also a materialized view + case _: ViewAlreadyExistsException if allowExisting => None + } + } + +} diff --git a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala index 6dd1188b78e8..a1e0b5ff68b5 100644 --- a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala +++ b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropV2ViewExec.scala @@ -18,6 +18,11 @@ */ package org.apache.spark.sql.execution.datasources.v2 +import org.apache.iceberg.catalog.Namespace +import org.apache.iceberg.catalog.TableIdentifier +import org.apache.iceberg.exceptions +import org.apache.iceberg.spark.SparkCatalog +import org.apache.iceberg.view.View import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.NoSuchViewException import org.apache.spark.sql.catalyst.expressions.Attribute @@ -30,6 +35,32 @@ case class DropV2ViewExec(catalog: ViewCatalog, ident: Identifier, ifExists: Boo override lazy val output: Seq[Attribute] = Nil override protected def run(): Seq[InternalRow] = { + // If the catalog is a SparkCatalog, check for materialized view storage table cleanup + catalog match { + case sparkCatalog: SparkCatalog => + val icebergViewCatalog = sparkCatalog.icebergViewCatalog() + var view: Option[View] = None + try { + val ns = Namespace.of(ident.namespace(): _*) + val viewId = TableIdentifier.of(ns, ident.name()) + view = Some(icebergViewCatalog.loadView(viewId)) + } catch { + case _: exceptions.NoSuchViewException => + if (!ifExists) { + throw new NoSuchViewException(ident) + } + } + // if view is a materialized view, drop the storage table first + view.foreach { v => + val storageTable = v.currentVersion().storageTable() + if (storageTable != null) { + val storageIdent = Identifier.of(storageTable.namespace().levels(), storageTable.name()) + sparkCatalog.dropTable(storageIdent) + } + } + case _ => // not a SparkCatalog, skip MV cleanup + } + val dropped = catalog.dropView(ident) if (!dropped && !ifExists) { throw new NoSuchViewException(ident) diff --git a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala index 7d46b2e04597..a92f190b18b7 100644 --- a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala +++ b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala @@ -36,6 +36,7 @@ import org.apache.spark.sql.catalyst.plans.logical.DropPartitionField import org.apache.spark.sql.catalyst.plans.logical.DropTag import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.plans.logical.OrderAwareCoalesce +import org.apache.spark.sql.catalyst.plans.logical.RefreshMaterializedViewStatement import org.apache.spark.sql.catalyst.plans.logical.RenameTable import org.apache.spark.sql.catalyst.plans.logical.ReplacePartitionField import org.apache.spark.sql.catalyst.plans.logical.SetIdentifierFields @@ -135,6 +136,35 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi case DropIcebergView(ResolvedIdentifier(viewCatalog: ViewCatalog, ident), ifExists) => DropV2ViewExec(viewCatalog, ident, ifExists) :: Nil + case CreateIcebergView( + ResolvedIdentifier(viewCatalog: ViewCatalog, ident), + queryText, + query, + columnAliases, + columnComments, + comment, + _, + properties, + allowExisting, + replace, + _, + Some(materializedViewOptions), + _, + _) => + CreateMaterializedViewExec( + catalog = viewCatalog, + ident = ident, + queryText = queryText, + columnAliases = columnAliases, + columnComments = columnComments, + queryColumnNames = query.schema.fieldNames.toIndexedSeq, + viewSchema = query.schema, + comment = comment, + properties = properties, + allowExisting = allowExisting, + replace = replace, + storageTableIdentifier = materializedViewOptions.storageTableIdentifier) :: Nil + case CreateIcebergView( ResolvedIdentifier(viewCatalog: ViewCatalog, ident), queryText, @@ -147,6 +177,7 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi allowExisting, replace, viewSchemaMode, + None, _, _) => CreateV2ViewExec( @@ -186,6 +217,9 @@ case class ExtendedDataSourceV2Strategy(spark: SparkSession) extends Strategy wi case UnsetViewProperties(ResolvedV2View(catalog, ident, _), propertyKeys, ifExists) => IcebergAlterV2ViewUnsetPropertiesExec(catalog, ident, propertyKeys, ifExists) :: Nil + case RefreshMaterializedViewStatement(catalog, ident) => + RefreshMaterializedViewExec(catalog, ident) :: Nil + case _ => Nil } diff --git a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala new file mode 100644 index 000000000000..4899c8f9f730 --- /dev/null +++ b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.v2 + +import org.apache.iceberg.catalog.Namespace +import org.apache.iceberg.catalog.TableIdentifier +import org.apache.iceberg.relocated.com.google.common.base.Preconditions +import org.apache.iceberg.spark.SparkCatalog +import org.apache.iceberg.spark.source.HasIcebergCatalog +import org.apache.iceberg.spark.source.SparkTable +import org.apache.iceberg.spark.source.SparkView +import org.apache.iceberg.view.RefreshState +import org.apache.iceberg.view.RefreshStateParser +import org.apache.iceberg.view.SourceTableState +import org.apache.iceberg.view.SourceViewState +import org.apache.iceberg.view.SQLViewRepresentation +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.NoSuchTableException +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.ViewCatalog +import org.apache.spark.sql.functions +import scala.jdk.CollectionConverters._ + +case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) + extends LeafV2CommandExec { + + override def output: Seq[Attribute] = Nil + + override protected def run(): Seq[InternalRow] = { + val sparkCatalog = catalog.asInstanceOf[SparkCatalog] + val icebergViewCatalog = sparkCatalog.icebergViewCatalog() + val viewId = TableIdentifier.of(Namespace.of(ident.namespace(): _*), ident.name()) + val view = icebergViewCatalog.loadView(viewId) + + val storageTableId = view.currentVersion().storageTable() + Preconditions.checkState( + storageTableId != null, + "Cannot refresh %s: not a materialized view (no storage table)", + ident) + + // Extract the SQL query from the view's representations + val sparkSql = view + .currentVersion() + .representations() + .asScala + .collect { case sql: SQLViewRepresentation if sql.dialect() == "spark" => sql.sql() } + .headOption + .getOrElse(throw new IllegalStateException( + s"Cannot refresh $ident: no Spark SQL representation found")) + + val refreshStartTimestampMs = System.currentTimeMillis() + + // Execute the view's query to get the current result set. Each view column then takes its + // values from the query column of the same name, not from the query column in the same + // position, which is how the view itself is read: the names the query produced are recorded + // when the view is created, and each view column is paired with the name recorded for it. + val viewColumnNames = view.schema().columns().asScala.map(_.name()).toSeq + val recordedQueryColumnNames = + SparkView.toView(sparkCatalog.name(), view).queryColumnNames().toSeq + Preconditions.checkState( + recordedQueryColumnNames.isEmpty + || recordedQueryColumnNames.length == viewColumnNames.length, + "Cannot refresh %s: view has %s column(s) but %s query column name(s) are recorded", + ident, + Int.box(viewColumnNames.length), + Int.box(recordedQueryColumnNames.length)) + + // A view created outside Spark records no query column names, and such a view is read by + // looking up its own column names in the query's output, so a refresh does the same. + val queryColumnNames = + if (recordedQueryColumnNames.isEmpty) viewColumnNames else recordedQueryColumnNames + + val rawQueryResult = session.sql(sparkSql) + // Either set of names can be missing from the query's output. Recorded names go stale when a + // source column is renamed or dropped: a view created as SELECT * FROM t over a table t of + // (id, data) records id for its first column, so renaming t.id to ident makes the query + // produce (ident, data), leaving no id column to read. The view's own names, used when none + // were recorded, may never have matched the query's output at all. Spark reports an + // incompatible schema change when reading a view in either state, so a refresh fails here + // rather than materializing what the view cannot read. + val missingColumns = + queryColumnNames.filterNot(rawQueryResult.schema.fieldNames.toSet.contains) + Preconditions.checkState( + missingColumns.isEmpty, + "Cannot refresh %s: query does not produce column(s) [%s] that the view reads. " + + "Recreate the view to match the current query.", + ident, + missingColumns.mkString(", ")) + + val queryResult = rawQueryResult.select(queryColumnNames.zip(viewColumnNames).map { + case (queryColumn, viewColumn) => functions.col(queryColumn).as(viewColumn) + }: _*) + + // Discover source tables and views from the query's logical plan and capture their + // current state + val sourceStates = collectSourceStates(queryResult.queryExecution.analyzed) + + // Build refresh state + val refreshState = new RefreshState( + view.currentVersion().versionId(), + sourceStates.asJava, + refreshStartTimestampMs) + val refreshStateJson = RefreshStateParser.toJson(refreshState) + + // Write results to storage table, replacing existing data + val storageTableRef = String.format( + "%s.%s.%s", + sparkCatalog.name(), + storageTableId.namespace().toString, + storageTableId.name()) + try { + queryResult + .writeTo(storageTableRef) + .option("snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) + .overwrite(functions.lit(true)) + } catch { + case e: NoSuchTableException => + throw new IllegalStateException( + s"Storage table $storageTableRef not found during refresh", + e) + } + + Nil + } + + /** + * Returns the catalog name to record for a source, or null when the source is in the + * materialized view's own catalog. + * + *

A null catalog keeps refresh state portable: a materialized view and its sources that live + * in the same catalog stay resolvable when that catalog is registered under a different name. + */ + private def sourceCatalogName(sourceCatalog: HasIcebergCatalog): String = { + if (sourceCatalog.name() == catalog.name()) null else sourceCatalog.name() + } + + private def collectSourceStates(plan: org.apache.spark.sql.catalyst.plans.logical.LogicalPlan) + : List[org.apache.iceberg.view.SourceState] = { + val seen = scala.collection.mutable.LinkedHashSet.empty[String] + val states = scala.collection.mutable.ListBuffer.empty[org.apache.iceberg.view.SourceState] + + // Every leaf relation backed by an Iceberg catalog is a candidate, including relations from + // catalogs other than the one holding the materialized view. Matching on HasIcebergCatalog + // rather than a concrete catalog class also covers Iceberg tables reached through the session + // catalog. Relations are deduplicated by catalog name and identifier so that a table + // referenced more than once yields a single state. + // + // Only a SparkTable is recorded. Such a relation carries the snapshot that was resolved when + // it was analyzed, and its scan is pinned to that snapshot, so the identity, snapshot, and + // branch are read from the relation itself rather than from a second load of the table. The + // other relations an Iceberg catalog can produce are left out on purpose, following the + // strategy of recording only the Iceberg dependencies a producer can track: a V1Table reached + // through the session catalog is not an Iceberg table, and a SparkChangelogTable reads a range + // of snapshots rather than a single one, so neither has a snapshot id to record. Leaving a + // dependency untracked means its changes do not make this materialized view stale. + plan.collectLeaves().foreach { + case r: org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation + if r.catalog.exists(_.isInstanceOf[HasIcebergCatalog]) && r.identifier.isDefined => + r.table match { + case sparkTable: SparkTable => + val sourceCatalog = r.catalog.get.asInstanceOf[HasIcebergCatalog] + val tableIdent = r.identifier.get + val key = "table:" + sourceCatalog.name() + "." + tableIdent.toString + if (seen.add(key)) { + val icebergId = sourceCatalog.icebergIdentifier(tableIdent) + val pinnedSnapshotId = sparkTable.snapshotId() + states += new SourceTableState( + icebergId.name(), + icebergId.namespace().levels().toList.asJava, + sourceCatalogName(sourceCatalog), + sparkTable.table().uuid().toString, + if (pinnedSnapshotId != null) { + pinnedSnapshotId.longValue() + } else { + RefreshState.NO_SNAPSHOT_ID + }, + sparkTable.branch()) + } + + case _ => // not an Iceberg table with a single snapshot, so not tracked + } + case _ => // skip non-iceberg leaves + } + + // Spark's analyzer replaces every view reference with a View node wrapping the view's + // expanded query, including transitively for view-of-view chains, so a single pass over + // the whole plan (not just its leaves) discovers every source view at every nesting depth. + // Matching View nodes rather than the SubqueryAlias that wraps them keeps tables out of + // this pass, since Spark aliases table references the same way. + plan + .collect { case view: org.apache.spark.sql.catalyst.plans.logical.View => view.desc } + .foreach { desc => + val viewIdent = desc.identifier + viewIdent.catalog.foreach { catalogName => + val key = "view:" + catalogName + "." + viewIdent.unquotedString + if (seen.add(key)) { + val icebergId = + TableIdentifier.of(Namespace.of(viewIdent.database.toList: _*), viewIdent.table) + // The catalog is matched positively, mirroring the table pass: a catalog that is not + // backed by Iceberg, or one that cannot serve views, has no view state to record. A + // NoSuchViewException means the name resolves to something other than an Iceberg view, + // such as a Spark view served by the session catalog, so it is not tracked either. + // Any other failure to load an Iceberg view is left to propagate, because recording + // an incomplete set of sources would make this materialized view look fresher than it + // is rather than fail the refresh. + session.sessionState.catalogManager.catalog(catalogName) match { + case sourceCatalog: HasIcebergCatalog => + val icebergViewCatalog = sourceCatalog.icebergViewCatalog() + if (icebergViewCatalog != null) { + try { + val view = icebergViewCatalog.loadView(icebergId) + states += new SourceViewState( + icebergId.name(), + icebergId.namespace().levels().toList.asJava, + sourceCatalogName(sourceCatalog), + view.uuid().toString, + view.currentVersion().versionId()) + } catch { + case _: org.apache.iceberg.exceptions.NoSuchViewException => // not tracked + } + } + + case _ => // not an Iceberg catalog, so not tracked + } + } + } + } + + states.toList + } + + override def simpleString(maxFields: Int): String = { + s"RefreshMaterializedViewExec: ${ident}" + } +} diff --git a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java new file mode 100644 index 000000000000..7c992a6d2bdc --- /dev/null +++ b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -0,0 +1,1040 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.extensions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Map; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.ParameterizedTestExtension; +import org.apache.iceberg.Parameters; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.RuntimeIOException; +import org.apache.iceberg.inmemory.InMemoryCatalog; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.spark.MaterializedViewUtil; +import org.apache.iceberg.spark.SparkCatalog; +import org.apache.iceberg.spark.SparkCatalogConfig; +import org.apache.iceberg.spark.SparkSessionCatalog; +import org.apache.iceberg.spark.source.SparkMaterializedView; +import org.apache.iceberg.view.RefreshState; +import org.apache.iceberg.view.RefreshStateParser; +import org.apache.iceberg.view.SourceTableState; +import org.apache.iceberg.view.SourceViewState; +import org.apache.iceberg.view.View; +import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; +import org.apache.spark.sql.catalyst.analysis.NoSuchViewException; +import org.apache.spark.sql.connector.catalog.CatalogPlugin; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.RelationCatalog; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.connector.catalog.ViewCatalog; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(ParameterizedTestExtension.class) +public class TestMaterializedViews extends ExtensionsTestBase { + private static final String QUERY_COLUMN_NAMES = "spark.query-column-names"; + + private static final Namespace NAMESPACE = Namespace.of("default"); + private final String tableName = "table"; + private final String materializedViewName = "materialized_view"; + + @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}") + protected static Object[][] parameters() { + Map properties = + Maps.newHashMap(SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.properties()); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, "file:" + getTempWarehouseDir()); + properties.put(CatalogProperties.CATALOG_IMPL, InMemoryCatalogWithLocalFileIO.class.getName()); + return new Object[][] { + { + SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.catalogName(), + SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.implementation(), + properties + } + }; + } + + private static String getTempWarehouseDir() { + try { + File tempDir = Files.createTempDirectory("warehouse-").toFile(); + tempDir.deleteOnExit(); + return tempDir.getAbsolutePath(); + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @BeforeEach + @Override + public void before() { + // Set up a simple InMemoryCatalog as validation catalog to avoid base class + // configureValidationCatalog() failing on our custom catalog-impl. + this.validationCatalog = new InMemoryCatalog(); + this.validationNamespaceCatalog = + (org.apache.iceberg.catalog.SupportsNamespaces) validationCatalog; + + spark.conf().set("spark.sql.catalog." + catalogName, implementation); + catalogConfig.forEach( + (key, value) -> spark.conf().set("spark.sql.catalog." + catalogName + "." + key, value)); + + sql("CREATE NAMESPACE IF NOT EXISTS default"); + spark.conf().set("spark.sql.defaultCatalog", catalogName); + sql("USE %s", catalogName); + sql("CREATE NAMESPACE IF NOT EXISTS %s", NAMESPACE); + sql("CREATE TABLE %s (id INT, data STRING)", tableName); + } + + @AfterEach + public void removeTable() { + sql("USE %s", catalogName); + sql("DROP VIEW IF EXISTS %s", materializedViewName); + sql("DROP VIEW IF EXISTS %s", "source_view"); + sql("DROP TABLE IF EXISTS %s", tableName); + } + + @TestTemplate + public void testStorageTableFieldOnViewVersion() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + View view = loadIcebergView(); + // storage-table should be set on the view version, not as a property + assertThat(view.currentVersion().storageTable()).isNotNull(); + assertThat(view.currentVersion().storageTable().name()) + .isEqualTo(materializedViewName + "__storage"); + assertThat(view.currentVersion().storageTable().namespace()).isEqualTo(NAMESPACE); + } + + @TestTemplate + public void testCreateOrReplaceViewOverMaterializedViewIsRejected() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + assertThatThrownBy( + () -> + sql( + "CREATE OR REPLACE VIEW %s AS SELECT id FROM %s", + materializedViewName, tableName)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Cannot replace materialized view") + .hasMessageContaining("drop the materialized view and create it again"); + + // The materialized view is untouched: it still references its storage table. + assertThat(loadIcebergView().currentVersion().storageTable()).isNotNull(); + } + + @TestTemplate + public void testQueryColumnNamesUseTheViewPropertyKeys() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + Map props = loadIcebergView().properties(); + assertThat(props).doesNotContainKey("queryColumnNames"); + assertThat(props).containsKey("spark.query-column-names-json"); + assertThat(props).containsEntry("spark.query-column-names", "id,data"); + } + + @TestTemplate + public void testRefreshFailsWhenQueryNoLongerProducesBoundColumns() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); + sql( + "CREATE MATERIALIZED VIEW %s (first, second) AS SELECT * FROM %s", + materializedViewName, tableName); + + // Renaming a source column leaves "first" paired with a name the query no longer produces. + // Reading a plain view in this state reports an incompatible schema change, so the refresh + // reports the columns it cannot read rather than filling them from the query's output order. + sql("ALTER TABLE %s RENAME COLUMN id TO ident", tableName); + + assertThatThrownBy(() -> sql("REFRESH MATERIALIZED VIEW %s", materializedViewName)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("query does not produce column(s) [id] that the view reads"); + } + + @TestTemplate + public void testRefreshReadsQueryColumnsByName() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); + sql( + "CREATE MATERIALIZED VIEW %s (first, second) AS SELECT * FROM %s", + materializedViewName, tableName); + + // Reordering the source columns changes the order of the query's output, but each view + // column takes its values from the query column it was paired with, so "first" still reads id. + sql("ALTER TABLE %s ALTER COLUMN data FIRST", tableName); + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + assertThat(sql("SELECT first FROM %s", materializedViewName)) + .containsExactlyInAnyOrder(row(1), row(2), row(3)); + } + + @TestTemplate + public void testRefreshUsesViewColumnNamesWhenQueryColumnNamesAreNotRecorded() { + sql("DROP TABLE IF EXISTS source_table"); + sql("CREATE TABLE source_table (x STRING, y STRING)"); + sql("INSERT INTO source_table VALUES ('x1', 'y1'), ('x2', 'y2')"); + + // A materialized view created outside Spark records no query column names, because that is a + // Spark property. Such a view is read by resolving its own column names against the query. + org.apache.iceberg.Schema schema = + new org.apache.iceberg.Schema( + org.apache.iceberg.types.Types.NestedField.optional( + 1, "x", org.apache.iceberg.types.Types.StringType.get()), + org.apache.iceberg.types.Types.NestedField.optional( + 2, "y", org.apache.iceberg.types.Types.StringType.get())); + sql("CREATE TABLE external_mv__storage (x STRING, y STRING)"); + sparkCatalog() + .icebergViewCatalog() + .buildView(TableIdentifier.of(NAMESPACE, "external_mv")) + .withQuery("spark", "SELECT * FROM source_table") + .withDefaultNamespace(NAMESPACE) + .withDefaultCatalog(catalogName) + .withSchema(schema) + .withStorageTableIdentifier(TableIdentifier.of(NAMESPACE, "external_mv__storage")) + .create(); + + assertThat(loadIcebergView("external_mv").properties()) + .doesNotContainKey("spark.query-column-names"); + + sql("REFRESH MATERIALIZED VIEW external_mv"); + assertThat(sql("SELECT x, y FROM external_mv ORDER BY x")) + .containsExactly(row("x1", "y1"), row("x2", "y2")); + + // Reordering the source columns must not change which column each view column reads, so the + // materialized view keeps agreeing with the view that it materializes. + sql("ALTER TABLE source_table ALTER COLUMN y FIRST"); + sql("REFRESH MATERIALIZED VIEW external_mv"); + assertThat(sql("SELECT x, y FROM external_mv ORDER BY x")) + .containsExactly(row("x1", "y1"), row("x2", "y2")); + + sql("DROP TABLE IF EXISTS source_table"); + } + + private void assertMaterializedViewMatchesView( + String materializedView, + String view, + String firstColumn, + String secondColumn, + boolean expectedFresh, + Object[]... expectedRows) { + boolean fresh; + try { + fresh = + sparkTableCatalog() + .loadTable(Identifier.of(new String[] {NAMESPACE.toString()}, materializedView)) + instanceof SparkMaterializedView; + } catch (NoSuchTableException e) { + fresh = false; + } + + assertThat(fresh) + .as("%s should be %s", materializedView, expectedFresh ? "fresh" : "stale") + .isEqualTo(expectedFresh); + + String query = "SELECT %s, %s FROM %s ORDER BY %s"; + assertThat(sql(query, firstColumn, secondColumn, materializedView, firstColumn)) + .as("%s should read the columns its own schema names", materializedView) + .containsExactly(expectedRows); + assertThat(sql(query, firstColumn, secondColumn, view, firstColumn)) + .as("%s should agree with the view that %s materializes", view, materializedView) + .containsExactly(expectedRows); + } + + /** + * A materialized view must return what the view it materializes returns, whether it is read from + * its storage table or from its query, and whether or not Spark recorded the query's column + * names. Reordering the source table's columns must not change any of that. + */ + @TestTemplate + public void testMaterializedViewMatchesViewAcrossFreshnessAndSourceReorder() { + sql("DROP TABLE IF EXISTS src"); + sql("CREATE TABLE src (x STRING, y STRING)"); + sql("INSERT INTO src VALUES ('x1', 'y1')"); + + // Created through Spark, so the query's column names are recorded. The aliases differ from + // the query's column names, so a refresh that ignored the recorded names would be visible. + sql("CREATE VIEW v_named (a, b) AS SELECT * FROM src"); + sql("CREATE MATERIALIZED VIEW mv_named (a, b) AS SELECT * FROM src"); + + // Created outside Spark, so no query column names are recorded. + org.apache.iceberg.Schema schema = + new org.apache.iceberg.Schema( + org.apache.iceberg.types.Types.NestedField.optional( + 1, "x", org.apache.iceberg.types.Types.StringType.get()), + org.apache.iceberg.types.Types.NestedField.optional( + 2, "y", org.apache.iceberg.types.Types.StringType.get())); + sql("CREATE TABLE mv_unnamed__storage (x STRING, y STRING)"); + sparkCatalog() + .icebergViewCatalog() + .buildView(TableIdentifier.of(NAMESPACE, "mv_unnamed")) + .withQuery("spark", "SELECT * FROM src") + .withDefaultNamespace(NAMESPACE) + .withDefaultCatalog(catalogName) + .withSchema(schema) + .withStorageTableIdentifier(TableIdentifier.of(NAMESPACE, "mv_unnamed__storage")) + .create(); + sparkCatalog() + .icebergViewCatalog() + .buildView(TableIdentifier.of(NAMESPACE, "v_unnamed")) + .withQuery("spark", "SELECT * FROM src") + .withDefaultNamespace(NAMESPACE) + .withDefaultCatalog(catalogName) + .withSchema(schema) + .create(); + + assertThat(loadIcebergView("mv_named").properties()).containsEntry(QUERY_COLUMN_NAMES, "x,y"); + assertThat(loadIcebergView("mv_unnamed").properties()).doesNotContainKey(QUERY_COLUMN_NAMES); + + sql("REFRESH MATERIALIZED VIEW mv_named"); + sql("REFRESH MATERIALIZED VIEW mv_unnamed"); + assertMaterializedViewMatchesView("mv_named", "v_named", "a", "b", true, row("x1", "y1")); + assertMaterializedViewMatchesView("mv_unnamed", "v_unnamed", "x", "y", true, row("x1", "y1")); + + // Writing to the source makes both materialized views stale, so they are read from their + // queries rather than from their storage tables. + sql("INSERT INTO src VALUES ('x2', 'y2')"); + assertMaterializedViewMatchesView( + "mv_named", "v_named", "a", "b", false, row("x1", "y1"), row("x2", "y2")); + assertMaterializedViewMatchesView( + "mv_unnamed", "v_unnamed", "x", "y", false, row("x1", "y1"), row("x2", "y2")); + + // Reordering the source's columns does not write a snapshot, so both materialized views stay + // fresh and keep serving the rows their storage tables already hold. + sql("REFRESH MATERIALIZED VIEW mv_named"); + sql("REFRESH MATERIALIZED VIEW mv_unnamed"); + sql("ALTER TABLE src ALTER COLUMN y FIRST"); + assertMaterializedViewMatchesView( + "mv_named", "v_named", "a", "b", true, row("x1", "y1"), row("x2", "y2")); + assertMaterializedViewMatchesView( + "mv_unnamed", "v_unnamed", "x", "y", true, row("x1", "y1"), row("x2", "y2")); + + sql("INSERT INTO src (x, y) VALUES ('x3', 'y3')"); + assertMaterializedViewMatchesView( + "mv_named", "v_named", "a", "b", false, row("x1", "y1"), row("x2", "y2"), row("x3", "y3")); + assertMaterializedViewMatchesView( + "mv_unnamed", + "v_unnamed", + "x", + "y", + false, + row("x1", "y1"), + row("x2", "y2"), + row("x3", "y3")); + + // Refreshing after the reorder looks the recorded names up in the query's output again, now + // that the query's output order no longer matches the order the columns were paired in. + sql("REFRESH MATERIALIZED VIEW mv_named"); + sql("REFRESH MATERIALIZED VIEW mv_unnamed"); + assertMaterializedViewMatchesView( + "mv_named", "v_named", "a", "b", true, row("x1", "y1"), row("x2", "y2"), row("x3", "y3")); + assertMaterializedViewMatchesView( + "mv_unnamed", + "v_unnamed", + "x", + "y", + true, + row("x1", "y1"), + row("x2", "y2"), + row("x3", "y3")); + + sql("DROP VIEW IF EXISTS mv_named"); + sql("DROP VIEW IF EXISTS mv_unnamed"); + sql("DROP VIEW IF EXISTS v_named"); + sql("DROP VIEW IF EXISTS v_unnamed"); + sql("DROP TABLE IF EXISTS src"); + } + + @TestTemplate + public void testStaleReadReadsQueryColumnsByName() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); + sql( + "CREATE MATERIALIZED VIEW %s (first, second) AS SELECT * FROM %s", + materializedViewName, tableName); + + // Reordering the source columns changes the order of the query's output. A stale materialized + // view is read by running that query, and each of its columns takes its values from the query + // column it was paired with, so "first" still reads id. + sql("ALTER TABLE %s ALTER COLUMN data FIRST", tableName); + + assertThat(sql("SELECT first, second FROM %s ORDER BY first", materializedViewName)) + .containsExactly(row(1, "a"), row(2, "b"), row(3, "c")); + } + + @TestTemplate + public void testColumnAliasesAreRespectedWhenStaleAndWhenFresh() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); + sql( + "CREATE MATERIALIZED VIEW %s (first, second) AS SELECT id, data FROM %s", + materializedViewName, tableName); + String storageTableName = materializedViewName + "__storage"; + + // A stale materialized view is read through its definition, so the query runs and its columns + // are named by the aliases. + assertThat(spark.table(materializedViewName).schema().fieldNames()) + .containsExactly("first", "second"); + assertThat(sql("SELECT first, second FROM %s ORDER BY first", materializedViewName)) + .containsExactly(row(1, "a"), row(2, "b"), row(3, "c")); + assertThat(analyzedPlan("SELECT first FROM " + materializedViewName)) + .contains("default." + tableName) + .doesNotContain(storageTableName); + + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // A fresh materialized view is read from its storage table instead of by running the query, + // so its plan is a relation scan that does not reference the source table. The columns keep + // the same names and values, so which of the two answers the query is not observable. + assertThat(spark.table(materializedViewName).schema().fieldNames()) + .containsExactly("first", "second"); + assertThat(sql("SELECT first, second FROM %s ORDER BY first", materializedViewName)) + .containsExactly(row(1, "a"), row(2, "b"), row(3, "c")); + assertThat(analyzedPlan("SELECT first FROM " + materializedViewName)) + .contains("RelationV2") + .doesNotContain("default." + tableName); + + // The storage table names its columns the same way when it is read on its own. + assertThat(sql("SELECT first, second FROM %s ORDER BY first", storageTableName)) + .containsExactly(row(1, "a"), row(2, "b"), row(3, "c")); + } + + private String analyzedPlan(String query) { + return spark.sql(query).queryExecution().analyzed().treeString(); + } + + @TestTemplate + public void testColumnAliasesNameTheViewColumns() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); + sql( + "CREATE MATERIALIZED VIEW %s (first, second COMMENT 'second column')" + + " AS SELECT id, data FROM %s", + materializedViewName, tableName); + + View view = loadIcebergView(); + assertThat(view.schema().columns()) + .map(org.apache.iceberg.types.Types.NestedField::name) + .containsExactly("first", "second"); + assertThat(view.schema().findField("second").doc()).isEqualTo("second column"); + + // The aliases name the view; the query keeps its own output column names. + assertThat(view.properties()).containsEntry("spark.query-column-names", "id,data"); + + // The storage table materializes the view's columns, so it carries the aliases too. + org.apache.iceberg.Table storageTable = + sparkCatalog().icebergCatalog().loadTable(view.currentVersion().storageTable()); + assertThat(storageTable.schema().columns()) + .map(org.apache.iceberg.types.Types.NestedField::name) + .containsExactly("first", "second"); + + // Refreshing writes the query, whose columns are named id and data, into a storage table + // whose columns are named by the aliases. + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + assertThat(sql("SELECT first, second FROM %s ORDER BY first", materializedViewName)) + .containsExactly(row(1, "a"), row(2, "b"), row(3, "c")); + } + + @TestTemplate + public void testCreateOrReplaceIsRejected() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + assertThatThrownBy( + () -> + sql( + "CREATE OR REPLACE MATERIALIZED VIEW %s AS SELECT id FROM %s", + materializedViewName, tableName)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Cannot replace materialized view") + .hasMessageContaining("Drop the materialized view and create it again"); + } + + @TestTemplate + public void testNeverRefreshedMvIsNotFresh() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // A newly created MV has no snapshots on its storage table, so it's not fresh. + // loadView should succeed (returns stale view) + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())) + .isInstanceOf(org.apache.spark.sql.connector.catalog.View.class); + } catch (NoSuchViewException e) { + fail("Materialized view not found"); + } + } + + @TestTemplate + public void testReadFromStorageTableWhenFresh() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + simulateRefresh(); + + // Fresh MV: loadTable should return SparkMaterializedView + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Fresh materialized view should be loadable as a table"); + } + + // Fresh MV: loadRelation routes to the storage table rather than the view definition + try { + assertThat(sparkRelationCatalog().loadRelation(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Fresh materialized view should be resolvable as a relation"); + } + + // Fresh MV: loadView still returns the view definition instead of signalling via an exception + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())) + .isInstanceOf(org.apache.spark.sql.connector.catalog.View.class); + } catch (NoSuchViewException e) { + fail("Materialized view not found"); + } + } + + @TestTemplate + public void testFallbackToViewWhenStale() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + simulateRefresh(); + + // Insert more data to invalidate the refresh + sql("INSERT INTO %s VALUES (3, 'c')", tableName); + + // Stale MV: loadView should return a plain Spark view (falls back to query execution) + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())) + .isInstanceOf(org.apache.spark.sql.connector.catalog.View.class); + } catch (NoSuchViewException e) { + fail("Stale materialized view should be loadable as a view"); + } + + // Stale MV: loadRelation routes to the view definition, not the storage table + try { + assertThat(sparkRelationCatalog().loadRelation(viewIdentifier())) + .isNotInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Stale materialized view should be resolvable as a relation"); + } + + // Stale MV: loadTable should not resolve to the MV's storage table + assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining(materializedViewName); + } + + @TestTemplate + public void testStorageTableCreatedBeforeMvMetadata() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // The storage table should exist + String storageTableName = + MaterializedViewUtil.getDefaultMaterializedViewStorageTableIdentifier( + Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName)) + .name(); + assertThat(sql("SHOW TABLES")) + .anySatisfy(row -> assertThat(row[1]).isEqualTo(storageTableName)); + } + + @TestTemplate + public void testDefaultStorageTableNaming() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // Default naming should be __storage + String expectedStorageTableName = materializedViewName + "__storage"; + assertThat(sql("SHOW TABLES")) + .anySatisfy(row -> assertThat(row[1]).isEqualTo(expectedStorageTableName)); + } + + @TestTemplate + public void testStoredAsClause() { + String customTableName = "custom_table_name"; + sql( + "CREATE MATERIALIZED VIEW %s STORED AS '%s' AS SELECT id, data FROM %s", + materializedViewName, customTableName, tableName); + + // Assert that the storage table with the custom name is in the list of tables + assertThat(sql("SHOW TABLES")).anySatisfy(row -> assertThat(row[1]).isEqualTo(customTableName)); + } + + @TestTemplate + public void testRefreshMaterializedView() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // Refresh the materialized view + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // After refresh, the MV should be fresh and loadable as a table + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Refreshed materialized view should be loadable as a table"); + } + + // Verify the storage table has data + View view = loadIcebergView(); + String storageTableRef = + String.format( + "%s.%s.%s", catalogName, NAMESPACE, view.currentVersion().storageTable().name()); + assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(2); + } + + @TestTemplate + public void testRefreshMaterializedViewUpdatesData() { + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + // First refresh + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // Insert more data + sql("INSERT INTO %s VALUES (3, 'c')", tableName); + + // Before second refresh, the MV should be stale + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())) + .isInstanceOf(org.apache.spark.sql.connector.catalog.View.class); + } catch (NoSuchViewException e) { + fail("Stale materialized view should be loadable as a view"); + } + + // Second refresh + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // After refresh, the MV should be fresh again + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Refreshed materialized view should be loadable as a table"); + } + + // Verify the storage table has all 3 rows + View view = loadIcebergView(); + String storageTableRef = + String.format( + "%s.%s.%s", catalogName, NAMESPACE, view.currentVersion().storageTable().name()); + assertThat(sql("SELECT * FROM %s", storageTableRef)).hasSize(3); + } + + @TestTemplate + public void testRefreshRecordsNestedViewState() { + String sourceViewName = "source_view"; + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE VIEW %s AS SELECT id, data FROM %s", sourceViewName, tableName); + sql( + "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", + materializedViewName, sourceViewName); + + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + View sourceView = loadIcebergView(sourceViewName); + RefreshState refreshState = loadRefreshState(); + + // The refresh state should record both the nested source view and the base table it + // resolves to, since the analyzed query plan fully expands the view chain. + assertThat(refreshState.sourceStates()).hasSize(2); + + SourceViewState viewState = + refreshState.sourceStates().stream() + .filter(SourceViewState.class::isInstance) + .map(SourceViewState.class::cast) + .findFirst() + .orElseGet(() -> fail("Refresh state should record the nested source view")); + assertThat(viewState.name()).isEqualTo(sourceViewName); + assertThat(viewState.namespace()).isEqualTo(Arrays.asList(NAMESPACE.levels())); + assertThat(viewState.uuid()).isEqualTo(sourceView.uuid().toString()); + assertThat(viewState.versionId()).isEqualTo(sourceView.currentVersion().versionId()); + + SourceTableState tableState = + refreshState.sourceStates().stream() + .filter(SourceTableState.class::isInstance) + .map(SourceTableState.class::cast) + .findFirst() + .orElseGet(() -> fail("Refresh state should record the underlying base table")); + assertThat(tableState.name()).isEqualTo(tableName); + + sql("DROP VIEW IF EXISTS %s", sourceViewName); + } + + @TestTemplate + public void testStaleWhenNestedViewChanges() { + String sourceViewName = "source_view"; + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); + sql("CREATE VIEW %s AS SELECT id, data FROM %s WHERE id <= 2", sourceViewName, tableName); + sql( + "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", + materializedViewName, sourceViewName); + + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // Freshly refreshed: loadable as a table + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Refreshed materialized view should be loadable as a table"); + } + + // Replace the nested view's definition without touching the base table. This bumps the + // source view's version but leaves the underlying base table's snapshot unchanged. + sql( + "CREATE OR REPLACE VIEW %s AS SELECT id, data FROM %s WHERE id <= 1", + sourceViewName, tableName); + + // The MV should now be stale because its nested source view changed versions, even + // though the underlying base table's snapshot did not change. + try { + assertThat(sparkViewCatalog().loadView(viewIdentifier())) + .isInstanceOf(org.apache.spark.sql.connector.catalog.View.class); + } catch (NoSuchViewException e) { + fail("Materialized view with a stale nested view should be loadable as a view"); + } + assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining(materializedViewName); + + sql("DROP VIEW IF EXISTS %s", sourceViewName); + } + + @TestTemplate + public void testStaleWhenSourceViewIsRecreated() { + String sourceViewName = "source_view"; + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName); + sql("CREATE VIEW %s AS SELECT id, data FROM %s WHERE id <= 2", sourceViewName, tableName); + sql( + "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", + materializedViewName, sourceViewName); + + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + int refreshedVersionId = loadIcebergView(sourceViewName).currentVersion().versionId(); + + // Drop and recreate the source view with an unrelated definition. Version ids restart at 1, + // so the recreated view reports the same version that the refresh recorded and only the + // view's UUID identifies it as a different view. + sql("DROP VIEW %s", sourceViewName); + sql("CREATE VIEW %s AS SELECT id, data FROM %s WHERE id > 2", sourceViewName, tableName); + assertThat(loadIcebergView(sourceViewName).currentVersion().versionId()) + .isEqualTo(refreshedVersionId); + + assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining(materializedViewName); + + sql("DROP VIEW IF EXISTS %s", sourceViewName); + } + + @TestTemplate + public void testStaleWhenEmptySourceTableIsRecreated() { + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, tableName); + + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + try { + assertThat(sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Refreshed materialized view should be loadable as a table"); + } + + // Drop and recreate the empty source table. The recorded and the current state both report + // no snapshot, so only the table's UUID identifies it as a different table. + sql("DROP TABLE %s", tableName); + sql("CREATE TABLE %s (id INT, data STRING)", tableName); + + assertThatThrownBy(() -> sparkTableCatalog().loadTable(viewIdentifier())) + .isInstanceOf(NoSuchTableException.class) + .hasMessageContaining(materializedViewName); + } + + @TestTemplate + public void testCrossCatalogSourceTable() { + String otherCatalogName = + "other_catalog_" + java.util.UUID.randomUUID().toString().replace("-", ""); + String sourceTableName = "cross_catalog_source"; + configureCatalog(otherCatalogName); + + sql("CREATE NAMESPACE IF NOT EXISTS %s.%s", otherCatalogName, NAMESPACE); + sql( + "CREATE TABLE %s.%s.%s (id INT, data STRING)", + otherCatalogName, NAMESPACE, sourceTableName); + sql( + "INSERT INTO %s.%s.%s VALUES (1, 'a'), (2, 'b')", + otherCatalogName, NAMESPACE, sourceTableName); + + sql( + "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s.%s.%s", + materializedViewName, otherCatalogName, NAMESPACE, sourceTableName); + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // The source table lives in another catalog, so the refresh must record it with that + // catalog name in order for freshness to be checkable at all. + RefreshState refreshState = loadRefreshState(); + assertThat(refreshState.sourceStates()).hasSize(1); + SourceTableState tableState = (SourceTableState) refreshState.sourceStates().get(0); + assertThat(tableState.name()).isEqualTo(sourceTableName); + assertThat(tableState.catalog()).isEqualTo(otherCatalogName); + + sql("DROP TABLE IF EXISTS %s.%s.%s", otherCatalogName, NAMESPACE, sourceTableName); + } + + @TestTemplate + public void testCrossCatalogSourceTableChangeMakesMvStale() { + String otherCatalogName = + "other_catalog_" + java.util.UUID.randomUUID().toString().replace("-", ""); + String sourceTableName = "cross_catalog_source"; + configureCatalog(otherCatalogName); + + sql("CREATE NAMESPACE IF NOT EXISTS %s.%s", otherCatalogName, NAMESPACE); + sql( + "CREATE TABLE %s.%s.%s (id INT, data STRING)", + otherCatalogName, NAMESPACE, sourceTableName); + sql( + "INSERT INTO %s.%s.%s VALUES (1, 'a'), (2, 'b')", + otherCatalogName, NAMESPACE, sourceTableName); + + sql( + "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s.%s.%s", + materializedViewName, otherCatalogName, NAMESPACE, sourceTableName); + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // Changing the cross-catalog source must make the materialized view stale. + sql("INSERT INTO %s.%s.%s VALUES (3, 'c')", otherCatalogName, NAMESPACE, sourceTableName); + + try { + assertThat(sparkRelationCatalog().loadRelation(viewIdentifier())) + .isNotInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Stale materialized view should be resolvable as a relation"); + } + + assertThat(sql("SELECT * FROM %s.%s.%s", catalogName, NAMESPACE, materializedViewName)) + .hasSize(3); + + sql("DROP TABLE IF EXISTS %s.%s.%s", otherCatalogName, NAMESPACE, sourceTableName); + } + + private void configureCatalog(String name) { + Map properties = + Maps.newHashMap(SparkCatalogConfig.SPARK_WITH_MATERIALIZED_VIEWS.properties()); + properties.put(CatalogProperties.WAREHOUSE_LOCATION, "file:" + getTempWarehouseDir()); + properties.put(CatalogProperties.CATALOG_IMPL, InMemoryCatalogWithLocalFileIO.class.getName()); + spark.conf().set("spark.sql.catalog." + name, implementation); + properties.forEach( + (key, value) -> spark.conf().set("spark.sql.catalog." + name + "." + key, value)); + } + + /** + * Verifies that a source reached through the session catalog is tracked. + * + *

SparkSessionCatalog is a sibling of SparkCatalog rather than a subclass, so a type test + * against SparkCatalog would skip these sources at refresh, leaving the materialized view with no + * recorded sources and therefore permanently fresh. + */ + @TestTemplate + public void testSourceTableInSessionCatalog() { + String sourceTableName = "session_catalog_source"; + configureSessionCatalog(); + + sql( + "CREATE TABLE IF NOT EXISTS spark_catalog.%s.%s (id INT, data STRING) USING iceberg", + NAMESPACE, sourceTableName); + sql("INSERT INTO spark_catalog.%s.%s VALUES (1, 'a'), (2, 'b')", NAMESPACE, sourceTableName); + + sql( + "CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM spark_catalog.%s.%s", + materializedViewName, NAMESPACE, sourceTableName); + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + RefreshState refreshState = loadRefreshState(); + assertThat(refreshState.sourceStates()).hasSize(1); + SourceTableState tableState = (SourceTableState) refreshState.sourceStates().get(0); + assertThat(tableState.name()).isEqualTo(sourceTableName); + assertThat(tableState.catalog()).isEqualTo("spark_catalog"); + + // Changing the source in the session catalog must make the materialized view stale. + sql("INSERT INTO spark_catalog.%s.%s VALUES (3, 'c')", NAMESPACE, sourceTableName); + + try { + assertThat(sparkRelationCatalog().loadRelation(viewIdentifier())) + .isNotInstanceOf(SparkMaterializedView.class); + } catch (NoSuchTableException e) { + fail("Stale materialized view should be resolvable as a relation"); + } + + sql("DROP TABLE IF EXISTS spark_catalog.%s.%s", NAMESPACE, sourceTableName); + } + + private void configureSessionCatalog() { + spark.conf().set("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()); + spark.conf().set("spark.sql.catalog.spark_catalog.type", "hive"); + spark.conf().set("spark.sql.catalog.spark_catalog.default-namespace", "default"); + spark.conf().set("spark.sql.catalog.spark_catalog.cache-enabled", "false"); + } + + private RefreshState loadRefreshState() { + View view = loadIcebergView(); + org.apache.iceberg.catalog.TableIdentifier storageTableId = + view.currentVersion().storageTable(); + org.apache.iceberg.Table storageTable = + sparkCatalog().icebergCatalog().loadTable(storageTableId); + String refreshStateJson = + storageTable.currentSnapshot().summary().get(RefreshState.REFRESH_STATE_SUMMARY_KEY); + return RefreshStateParser.fromJson(refreshStateJson); + } + + private void simulateRefresh() { + View view = loadIcebergView(); + org.apache.iceberg.catalog.TableIdentifier storageTableId = + view.currentVersion().storageTable(); + + org.apache.iceberg.Table baseTable = + sparkCatalog().icebergCatalog().loadTable(TableIdentifier.of(NAMESPACE, tableName)); + + // Get the base table's current snapshot ID + long baseSnapshotId = + (Long) + sql( + "SELECT snapshot_id FROM %s.%s.%s.snapshots ORDER BY committed_at DESC LIMIT 1", + catalogName, NAMESPACE, tableName) + .get(0)[0]; + + // Build refresh state matching the current view version and source table state + RefreshState refreshState = + new RefreshState( + view.currentVersion().versionId(), + Arrays.asList( + new SourceTableState( + tableName, + Arrays.asList(NAMESPACE.levels()), + null, + baseTable.uuid().toString(), + baseSnapshotId, + null)), + System.currentTimeMillis()); + String refreshStateJson = RefreshStateParser.toJson(refreshState); + + // Write data to storage table with refresh-state in the snapshot summary + String storageTableRef = + String.format("%s.%s.%s", catalogName, NAMESPACE, storageTableId.name()); + try { + spark + .sql(String.format("SELECT id, data FROM %s.%s.%s", catalogName, NAMESPACE, tableName)) + .writeTo(storageTableRef) + .option("snapshot-property." + RefreshState.REFRESH_STATE_SUMMARY_KEY, refreshStateJson) + .append(); + } catch (NoSuchTableException e) { + throw new RuntimeException("Storage table not found during simulated refresh", e); + } + } + + private ViewCatalog sparkViewCatalog() { + CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); + return (ViewCatalog) catalogPlugin; + } + + private TableCatalog sparkTableCatalog() { + CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); + return (TableCatalog) catalogPlugin; + } + + private RelationCatalog sparkRelationCatalog() { + CatalogPlugin catalogPlugin = spark.sessionState().catalogManager().catalog(catalogName); + return (RelationCatalog) catalogPlugin; + } + + private Identifier viewIdentifier() { + return Identifier.of(new String[] {NAMESPACE.toString()}, materializedViewName); + } + + private SparkCatalog sparkCatalog() { + return (SparkCatalog) spark.sessionState().catalogManager().catalog(catalogName); + } + + private View loadIcebergView() { + return loadIcebergView(materializedViewName); + } + + private View loadIcebergView(String viewName) { + org.apache.iceberg.catalog.ViewCatalog icebergViewCatalog = sparkCatalog().icebergViewCatalog(); + return icebergViewCatalog.loadView(TableIdentifier.of(NAMESPACE, viewName)); + } + + // Required to be public since it is loaded by org.apache.iceberg.CatalogUtil.loadCatalog + public static class InMemoryCatalogWithLocalFileIO extends InMemoryCatalog { + private FileIO localFileIO; + + @Override + public void initialize(String name, Map properties) { + super.initialize(name, properties); + localFileIO = new LocalFileIO(); + } + + @Override + protected TableOperations newTableOps(TableIdentifier tableIdentifier) { + return new InMemoryTableOperations(localFileIO, tableIdentifier); + } + + @Override + protected InMemoryCatalog.InMemoryViewOperations newViewOps(TableIdentifier identifier) { + return new InMemoryViewOperations(localFileIO, identifier); + } + } + + private static class LocalFileIO implements FileIO { + + private static String stripFilePrefix(String path) { + return path.startsWith("file:") ? path.substring(5) : path; + } + + @Override + public InputFile newInputFile(String path) { + return org.apache.iceberg.Files.localInput(stripFilePrefix(path)); + } + + @Override + public OutputFile newOutputFile(String path) { + String stripped = stripFilePrefix(path); + java.io.File parent = new java.io.File(stripped).getParentFile(); + if (!parent.isDirectory()) { + parent.mkdirs(); + } + return org.apache.iceberg.Files.localOutput(stripped); + } + + @Override + public void deleteFile(String path) { + if (!new File(stripFilePrefix(path)).delete()) { + throw new RuntimeIOException("Failed to delete file: " + path); + } + } + } +} diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java new file mode 100644 index 000000000000..a30c5f671176 --- /dev/null +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/MaterializedViewUtil.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark; + +import org.apache.spark.sql.connector.catalog.Identifier; + +public class MaterializedViewUtil { + + private MaterializedViewUtil() {} + + private static final String MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX = "__storage"; + + public static Identifier getDefaultMaterializedViewStorageTableIdentifier( + Identifier viewIdentifier) { + return Identifier.of( + viewIdentifier.namespace(), + viewIdentifier.name() + MATERIALIZED_VIEW_STORAGE_TABLE_IDENTIFIER_SUFFIX); + } +} diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java index 67c75c3a63d1..1345b45423a3 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java @@ -36,6 +36,7 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; import org.apache.iceberg.Transaction; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; @@ -56,12 +57,19 @@ import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.spark.actions.SparkActions; +import org.apache.iceberg.spark.source.HasIcebergCatalog; import org.apache.iceberg.spark.source.SparkChangelogTable; +import org.apache.iceberg.spark.source.SparkMaterializedView; import org.apache.iceberg.spark.source.SparkTable; import org.apache.iceberg.spark.source.SparkView; import org.apache.iceberg.spark.source.StagedSparkTable; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.view.RefreshState; +import org.apache.iceberg.view.RefreshStateParser; +import org.apache.iceberg.view.SourceState; +import org.apache.iceberg.view.SourceTableState; +import org.apache.iceberg.view.SourceViewState; import org.apache.iceberg.view.UpdateViewProperties; import org.apache.iceberg.view.ViewBuilder; import org.apache.iceberg.view.ViewProperties; @@ -73,8 +81,10 @@ import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException; import org.apache.spark.sql.catalyst.analysis.ViewAlreadyExistsException; import org.apache.spark.sql.catalyst.analysis.ViewUtil; +import org.apache.spark.sql.connector.catalog.CatalogPlugin; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.NamespaceChange; +import org.apache.spark.sql.connector.catalog.Relation; import org.apache.spark.sql.connector.catalog.StagedTable; import org.apache.spark.sql.connector.catalog.Table; import org.apache.spark.sql.connector.catalog.TableCatalog; @@ -564,17 +574,185 @@ public boolean viewExists(Identifier ident) { @Override public View loadView(Identifier ident) throws NoSuchViewException { if (null != asViewCatalog) { - try { - org.apache.iceberg.view.View icebergView = asViewCatalog.loadView(buildIdentifier(ident)); - return SparkView.toView(catalogName, icebergView); - } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { - throw new NoSuchViewException(ident); + org.apache.iceberg.view.View view = findIcebergView(ident); + if (null != view) { + return SparkView.toView(catalogName, view); } + + throw new NoSuchViewException(ident); } throw new NoSuchViewException(ident); } + /** + * Loads the Iceberg view for an identifier, or returns null when no such view exists. + * + *

Materialized view routing is decided by {@link #loadRelation(Identifier)}, which prefers the + * storage table for a fresh view, so this lookup reports absence instead of signalling control + * flow through exceptions. + */ + private org.apache.iceberg.view.View findIcebergView(Identifier ident) { + if (null == asViewCatalog) { + return null; + } + + try { + return asViewCatalog.loadView(buildIdentifier(ident)); + } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { + return null; + } + } + + /** + * Resolves an identifier that may name either a table or a view. + * + *

A materialized view is served from its storage table while it is fresh, and from its view + * definition otherwise, so the choice is made here rather than by having {@code loadView} signal + * the engine to retry with {@code loadTable}. + */ + @Override + public Relation loadRelation(Identifier ident) throws NoSuchTableException { + if (!isPathIdentifier(ident)) { + org.apache.iceberg.view.View view = findIcebergView(ident); + if (null != view && isMaterializedView(view)) { + return isFresh(view) + ? new SparkMaterializedView(catalogName, view, loadStorageTable(view)) + : SparkView.toView(catalogName, view); + } + } + + return super.loadRelation(ident); + } + + private boolean isMaterializedView(org.apache.iceberg.view.View view) { + return view.currentVersion().storageTable() != null; + } + + private org.apache.iceberg.catalog.TableIdentifier getStorageTableId( + org.apache.iceberg.view.View view) { + org.apache.iceberg.catalog.TableIdentifier storageTable = view.currentVersion().storageTable(); + Preconditions.checkState( + storageTable != null, "Storage table identifier is not set for materialized view."); + return storageTable; + } + + private Table loadStorageTable(org.apache.iceberg.view.View view) { + org.apache.iceberg.catalog.TableIdentifier storageTableId = getStorageTableId(view); + try { + Identifier sparkIdent = + Identifier.of(storageTableId.namespace().levels(), storageTableId.name()); + return loadTable(sparkIdent); + } catch (NoSuchTableException e) { + throw new IllegalStateException("Unable to load storage table for materialized view.", e); + } + } + + private boolean isFresh(org.apache.iceberg.view.View view) { + Table sparkStorageTable = loadStorageTable(view); + org.apache.iceberg.Table storageTable = ((SparkTable) sparkStorageTable).table(); + if (storageTable.currentSnapshot() == null) { + return false; + } + + String refreshStateJson = + storageTable.currentSnapshot().summary().get(RefreshState.REFRESH_STATE_SUMMARY_KEY); + if (refreshStateJson == null) { + return false; + } + + RefreshState refreshState = RefreshStateParser.fromJson(refreshStateJson); + + if (refreshState.viewVersionId() != view.currentVersion().versionId()) { + return false; + } + + for (SourceState sourceState : refreshState.sourceStates()) { + if (sourceState instanceof SourceTableState) { + if (!isSourceFresh((SourceTableState) sourceState)) { + return false; + } + } else if (sourceState instanceof SourceViewState) { + if (!isSourceFresh((SourceViewState) sourceState)) { + return false; + } + } + } + + return true; + } + + /** + * Returns whether a source table still matches the state captured by the last refresh. + * + *

A source is identified by its UUID as well as by its name because a table that was dropped + * and recreated, or replaced by an unrelated table with the same name, is not the table that was + * read. Snapshot ids alone cannot detect that: a recreated table restarts its history and an + * empty table records {@link RefreshState#NO_SNAPSHOT_ID} both before and after. + */ + private boolean isSourceFresh(SourceTableState tableState) { + try { + org.apache.iceberg.Table sourceTable = + sourceCatalog(tableState).loadTable(sourceIdentifier(tableState)); + if (!sourceTable.uuid().toString().equals(tableState.uuid())) { + return false; + } + + Snapshot snapshot = + tableState.ref() == null + ? sourceTable.currentSnapshot() + : sourceTable.snapshot(tableState.ref()); + long snapshotId = snapshot == null ? RefreshState.NO_SNAPSHOT_ID : snapshot.snapshotId(); + return snapshotId == tableState.snapshotId(); + } catch (Exception e) { + return false; + } + } + + /** + * Returns whether a source view still matches the state captured by the last refresh. + * + *

The UUID check is essential here: view version ids restart at 1, so a view that was dropped + * and recreated with an unrelated definition would otherwise report the recorded version. + */ + private boolean isSourceFresh(SourceViewState viewState) { + try { + org.apache.iceberg.view.View sourceView = + ((ViewCatalog) sourceCatalog(viewState)).loadView(sourceIdentifier(viewState)); + return sourceView.uuid().toString().equals(viewState.uuid()) + && sourceView.currentVersion().versionId() == viewState.versionId(); + } catch (Exception e) { + return false; + } + } + + /** + * Returns the Iceberg catalog that holds a source object. + * + *

A materialized view may read sources from a catalog other than its own, so a recorded + * catalog name is resolved through Spark's catalog manager rather than assumed to be this + * catalog. Callers treat a failure to resolve as "not fresh", since a source that cannot be + * reached cannot be shown to be unchanged. + */ + private Catalog sourceCatalog(SourceState sourceState) { + if (sourceState.catalog() == null) { + return icebergCatalog(); + } + + CatalogPlugin plugin = + SparkSession.active().sessionState().catalogManager().catalog(sourceState.catalog()); + Preconditions.checkArgument( + plugin instanceof HasIcebergCatalog, + "Cannot resolve source catalog %s: not an Iceberg catalog", + sourceState.catalog()); + return ((HasIcebergCatalog) plugin).icebergCatalog(); + } + + private TableIdentifier sourceIdentifier(SourceState sourceState) { + return TableIdentifier.of( + Namespace.of(sourceState.namespace().toArray(new String[0])), sourceState.name()); + } + @Override public View createView(Identifier ident, View view) throws ViewAlreadyExistsException, NoSuchNamespaceException { @@ -636,6 +814,8 @@ private View commitView(Identifier ident, View view, ViewCommit viewCommit) Map props = ViewUtil.createProperties(view); TableIdentifier viewIdentifier = buildIdentifier(ident); + checkNotMaterializedView(viewIdentifier, viewCommit); + try { ViewBuilder builder = asViewCatalog @@ -689,6 +869,28 @@ private View commitView(Identifier ident, View view, ViewCommit viewCommit) } } + private void checkNotMaterializedView(TableIdentifier viewIdentifier, ViewCommit viewCommit) { + if (viewCommit == ViewCommit.CREATE) { + return; + } + + org.apache.iceberg.view.View existingView; + try { + existingView = asViewCatalog.loadView(viewIdentifier); + } catch (org.apache.iceberg.exceptions.NoSuchViewException e) { + // there is nothing to replace, so the commit is a plain create + return; + } + + if (existingView.currentVersion().storageTable() != null) { + throw new UnsupportedOperationException( + String.format( + "Cannot replace materialized view %s with a view: " + + "drop the materialized view and create it again", + viewIdentifier)); + } + } + private Pair, org.apache.iceberg.view.View> createOrReplaceView( TableIdentifier viewIdentifier, ViewBuilder builder) { try { @@ -932,6 +1134,12 @@ private Table load(Identifier ident, TimeTravel timeTravel) throws NoSuchTableEx return loadPath((PathIdentifier) ident, timeTravel); } + // A fresh materialized view is readable as its storage table. + org.apache.iceberg.view.View view = findIcebergView(ident); + if (null != view && isMaterializedView(view) && isFresh(view)) { + return new SparkMaterializedView(catalogName, view, loadStorageTable(view)); + } + try { org.apache.iceberg.Table table = icebergCatalog.loadTable(buildIdentifier(ident)); return SparkTable.create(table, timeTravel); diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java new file mode 100644 index 000000000000..876a0493099b --- /dev/null +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkMaterializedView.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.source; + +import java.util.Set; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.view.View; +import org.apache.spark.sql.connector.catalog.SupportsRead; +import org.apache.spark.sql.connector.catalog.Table; +import org.apache.spark.sql.connector.catalog.TableCapability; +import org.apache.spark.sql.connector.read.ScanBuilder; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; + +/** + * A fresh materialized view exposed to Spark as a readable table. + * + *

Spark 4.2 represents views as an immutable {@code View} value produced by {@link + * SparkView#toView}, so a materialized view cannot extend it. Reads are served directly from the + * view's storage table while the Iceberg view metadata remains available for planning. + */ +public class SparkMaterializedView implements Table, SupportsRead { + private final String catalogName; + private final View icebergView; + private final Table storageTable; + + public SparkMaterializedView(String catalogName, View icebergView, Table storageTable) { + this.catalogName = catalogName; + this.icebergView = icebergView; + this.storageTable = storageTable; + } + + public View view() { + return icebergView; + } + + public String catalogName() { + return catalogName; + } + + public Table storageTable() { + return storageTable; + } + + @Override + public String name() { + return icebergView.name(); + } + + @Override + public StructType schema() { + return storageTable.schema(); + } + + @Override + public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) { + return ((SupportsRead) storageTable).newScanBuilder(options); + } + + @Override + public Set capabilities() { + return ImmutableSet.of(TableCapability.BATCH_READ); + } +} diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java index b20c87619ed8..36b4909e933e 100644 --- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java +++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java @@ -66,6 +66,16 @@ public enum SparkCatalogConfig { "spark_hive_with_views", SparkCatalog.class.getName(), ImmutableMap.of("type", "hive", "default-namespace", "default", "cache-enabled", "false")), + SPARK_WITH_MATERIALIZED_VIEWS( + "spark_with_mvs", + SparkCatalog.class.getName(), + ImmutableMap.of( + CatalogProperties.CATALOG_IMPL, + InMemoryCatalog.class.getName(), + "default-namespace", + "default", + "cache-enabled", + "false")), SPARK_SESSION_WITH_UNIQUE_LOCATION( "spark_catalog", SparkSessionCatalog.class.getName(), From bb460f9a9493ceef13d10b6afbf0ee677b8f3a37 Mon Sep 17 00:00:00 2001 From: wmoustafa <10084105+wmoustafa@users.noreply.github.com> Date: Sun, 20 Sep 2026 21:19:58 -0700 Subject: [PATCH 22/22] Spark 4.2: Record deep source state for fresh nested materialized views --- .../v2/RefreshMaterializedViewExec.scala | 88 ++++++ .../extensions/TestMaterializedViews.java | 273 ++++++++++++++++++ 2 files changed, 361 insertions(+) diff --git a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala index 4899c8f9f730..c7e035206af1 100644 --- a/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala +++ b/spark/v4.2/spark-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshMaterializedViewExec.scala @@ -23,6 +23,7 @@ import org.apache.iceberg.catalog.TableIdentifier import org.apache.iceberg.relocated.com.google.common.base.Preconditions import org.apache.iceberg.spark.SparkCatalog import org.apache.iceberg.spark.source.HasIcebergCatalog +import org.apache.iceberg.spark.source.SparkMaterializedView import org.apache.iceberg.spark.source.SparkTable import org.apache.iceberg.spark.source.SparkView import org.apache.iceberg.view.RefreshState @@ -243,9 +244,96 @@ case class RefreshMaterializedViewExec(catalog: ViewCatalog, ident: Identifier) } } + // A fresh nested materialized view is served from its storage table, so it reaches this plan + // as a SparkMaterializedView relation rather than as an expanded View node, and the two passes + // above skip it. Record it the same way the not-fresh (expanded) case is recorded: the nested + // materialized view itself as a source view, plus the deep sources it already captured in its + // own refresh state. A fresh materialized view's refresh state is a complete, flattened closure + // of its dependencies, so merging it once reproduces the full depth -- including materialized + // views nested further down -- without recomputing the query or recursing here. Entries are + // deduplicated by uuid against everything already recorded so a source reached through several + // paths is recorded once. + val recordedUuids = scala.collection.mutable.Set.empty[String] + recordedUuids ++= states.map(_.uuid()) + plan.collectLeaves().foreach { + case r: org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation + if r.catalog.exists(_.isInstanceOf[HasIcebergCatalog]) && r.identifier.isDefined => + r.table match { + case mv: SparkMaterializedView => + val sourceCatalog = r.catalog.get.asInstanceOf[HasIcebergCatalog] + val icebergId = sourceCatalog.icebergIdentifier(r.identifier.get) + if (recordedUuids.add(mv.view().uuid().toString)) { + states += new SourceViewState( + icebergId.name(), + icebergId.namespace().levels().toList.asJava, + sourceCatalogName(sourceCatalog), + mv.view().uuid().toString, + mv.view().currentVersion().versionId()) + } + mergeNestedRefreshState(mv, sourceCatalog.name(), recordedUuids, states) + + case _ => // only fresh materialized views are handled in this pass + } + case _ => // skip non-iceberg leaves + } + states.toList } + /** + * Merges the deep source states a fresh nested materialized view already recorded in its own + * refresh state into this refresh. A fresh materialized view's storage table always carries a + * refresh state, and that state is a complete, flattened closure of the materialized view's + * dependencies, so a single merge reproduces the full depth -- including any materialized views + * nested further down -- without recomputing the query or recursing. A never-refreshed storage + * table has no snapshot and therefore no state to merge. + */ + private def mergeNestedRefreshState( + mv: SparkMaterializedView, + childCatalogName: String, + recordedUuids: scala.collection.mutable.Set[String], + states: scala.collection.mutable.ListBuffer[org.apache.iceberg.view.SourceState]): Unit = { + val storageTable = mv.storageTable().asInstanceOf[SparkTable].table() + val snapshot = storageTable.currentSnapshot() + if (snapshot != null) { + val json = snapshot.summary().get(RefreshState.REFRESH_STATE_SUMMARY_KEY) + if (json != null) { + RefreshStateParser.fromJson(json).sourceStates().asScala.foreach { sourceState => + if (recordedUuids.add(sourceState.uuid())) { + states += rebaseCatalog(sourceState, childCatalogName) + } + } + } + } + } + + /** + * Rewrites a source state recorded by a nested materialized view so its catalog is expressed + * relative to the materialized view being refreshed. A nested state's null catalog means "the + * nested materialized view's own catalog", which is not this refresh's default, so it is resolved + * to an absolute name and then collapsed back to null only when it matches this materialized + * view's catalog. + */ + private def rebaseCatalog( + sourceState: org.apache.iceberg.view.SourceState, + childCatalogName: String): org.apache.iceberg.view.SourceState = { + val absolute = if (sourceState.catalog() == null) childCatalogName else sourceState.catalog() + val rebased = if (absolute == catalog.name()) null else absolute + sourceState match { + case table: SourceTableState => + new SourceTableState( + table.name(), + table.namespace(), + rebased, + table.uuid(), + table.snapshotId(), + table.ref()) + case view: SourceViewState => + new SourceViewState(view.name(), view.namespace(), rebased, view.uuid(), view.versionId()) + case other => other + } + } + override def simpleString(maxFields: Int): String = { s"RefreshMaterializedViewExec: ${ident}" } diff --git a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java index 7c992a6d2bdc..3ccc10eebb67 100644 --- a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java +++ b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMaterializedViews.java @@ -120,7 +120,17 @@ public void removeTable() { sql("USE %s", catalogName); sql("DROP VIEW IF EXISTS %s", materializedViewName); sql("DROP VIEW IF EXISTS %s", "source_view"); + // Nested materialized views used by the multi-level tests, dropped from the top of the chain + // down so a dropped view is never a source of one that still exists. + sql("DROP VIEW IF EXISTS %s", "middle_mv"); + sql("DROP VIEW IF EXISTS %s", "c_mv"); + sql("DROP VIEW IF EXISTS %s", "inner_mv"); + sql("DROP VIEW IF EXISTS %s", "g_mv"); + sql("DROP VIEW IF EXISTS %s", "stale_mv"); + sql("DROP VIEW IF EXISTS %s", "fresh_mv"); + sql("DROP VIEW IF EXISTS %s", "shared_child_mv"); sql("DROP TABLE IF EXISTS %s", tableName); + sql("DROP TABLE IF EXISTS %s", "fresh_base"); } @TestTemplate @@ -686,6 +696,269 @@ public void testRefreshRecordsNestedViewState() { sql("DROP VIEW IF EXISTS %s", sourceViewName); } + /** + * When a materialized view reads other materialized views that are themselves not fresh, both the + * refresh (producer) and the freshness check (consumer) treat the nested materialized views as + * ordinary views: the query is expanded through their definitions all the way down to the base + * table. + * + *

Refresh therefore records each nested materialized view as a source view (by + * version id) together with the base table the chain resolves to, and never records a nested + * materialized view's storage table. Because the nested materialized views are tracked by their + * view versions rather than by their storage tables, materializing one of them does not affect + * the parent's freshness, while a change to the base table does. + */ + @TestTemplate + public void testNestedMaterializedViewsAreExpandedAndTrackedAsViews() { + String innerMv = "inner_mv"; + String middleMv = "middle_mv"; + + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + + // A two-level chain of materialized views nested under the main materialized view. None of the + // nested materialized views is refreshed, so each is served from its view definition and the + // chain expands down to the base table. + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", innerMv, tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", middleMv, innerMv); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, middleMv); + + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // Producer: the refresh expanded the whole chain, recording the two nested materialized views + // as source views and the single base table as a source table. + RefreshState refreshState = loadRefreshState(); + assertThat(refreshState.sourceStates()).hasSize(3); + + Map viewStates = + refreshState.sourceStates().stream() + .filter(SourceViewState.class::isInstance) + .map(SourceViewState.class::cast) + .collect(java.util.stream.Collectors.toMap(SourceViewState::name, state -> state)); + assertThat(viewStates.keySet()).containsExactlyInAnyOrder(innerMv, middleMv); + assertThat(viewStates.get(innerMv).uuid()) + .isEqualTo(loadIcebergView(innerMv).uuid().toString()); + assertThat(viewStates.get(innerMv).versionId()) + .isEqualTo(loadIcebergView(innerMv).currentVersion().versionId()); + assertThat(viewStates.get(middleMv).uuid()) + .isEqualTo(loadIcebergView(middleMv).uuid().toString()); + assertThat(viewStates.get(middleMv).versionId()) + .isEqualTo(loadIcebergView(middleMv).currentVersion().versionId()); + + // Only the base table is recorded as a source table. No nested materialized view's storage + // table appears, which is what distinguishes view expansion from treating a nested + // materialized view as a table. + assertThat(refreshState.sourceStates()) + .filteredOn(SourceTableState.class::isInstance) + .singleElement() + .satisfies(state -> assertThat(((SourceTableState) state).name()).isEqualTo(tableName)); + + assertThat(materializedViewIsFresh(materializedViewName)) + .as("the main materialized view should be fresh after refresh") + .isTrue(); + + // Consumer treats nested materialized views as views: materializing a nested materialized view + // changes its storage table but not its view version, so the parent stays fresh. + sql("REFRESH MATERIALIZED VIEW %s", innerMv); + assertThat(materializedViewIsFresh(materializedViewName)) + .as("materializing a nested materialized view must not affect the parent's freshness") + .isTrue(); + + // Consumer expands deeply: changing the base table the chain resolves to makes the parent + // stale, proving that the base table's snapshot was recorded and is being checked. + sql("INSERT INTO %s VALUES (3, 'c')", tableName); + assertThat(materializedViewIsFresh(materializedViewName)) + .as("a base-table change must make the parent stale") + .isFalse(); + + sql("DROP VIEW IF EXISTS %s", materializedViewName); + sql("DROP VIEW IF EXISTS %s", middleMv); + sql("DROP VIEW IF EXISTS %s", innerMv); + } + + private boolean materializedViewIsFresh(String name) { + try { + return sparkTableCatalog().loadTable(Identifier.of(new String[] {NAMESPACE.toString()}, name)) + instanceof SparkMaterializedView; + } catch (NoSuchTableException e) { + return false; + } + } + + /** + * When a nested materialized view is fresh at the parent's refresh time it is served + * from its storage table, so the parent cannot expand it. Instead the refresh merges the nested + * materialized view's own recorded refresh state, which is a complete flattened closure of its + * dependencies. This makes the fresh case record the same shape as the not-fresh case — the + * nested materialized view's version plus the base table — and it composes across levels, so a + * materialized view nested two levels down still appears, all without recording any storage + * table. + */ + @TestTemplate + public void testFreshNestedMaterializedViewsMergeDeepStateAcrossLevels() { + String gMv = "g_mv"; + String cMv = "c_mv"; + + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + + // parent (materializedViewName) -> c_mv -> g_mv -> base table, with each nested materialized + // view refreshed so it is fresh (served from its storage table) when the level above it is + // refreshed. + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", gMv, tableName); + sql("REFRESH MATERIALIZED VIEW %s", gMv); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", cMv, gMv); + sql("REFRESH MATERIALIZED VIEW %s", cMv); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", materializedViewName, cMv); + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + // The parent recorded the full depth even though every nested materialized view was fresh and + // read from its storage table: both nested materialized views as source views (by version) and + // the single base table. + RefreshState refreshState = loadRefreshState(); + assertThat(refreshState.sourceStates()).hasSize(3); + + Map viewStates = + refreshState.sourceStates().stream() + .filter(SourceViewState.class::isInstance) + .map(SourceViewState.class::cast) + .collect(java.util.stream.Collectors.toMap(SourceViewState::name, state -> state)); + assertThat(viewStates.keySet()).containsExactlyInAnyOrder(cMv, gMv); + assertThat(viewStates.get(cMv).versionId()) + .isEqualTo(loadIcebergView(cMv).currentVersion().versionId()); + assertThat(viewStates.get(gMv).versionId()) + .isEqualTo(loadIcebergView(gMv).currentVersion().versionId()); + + // Only the base table is recorded as a source table; no nested materialized view's storage + // table appears, which is what makes this the "treat as view" shape rather than an opaque one. + assertThat(refreshState.sourceStates()) + .filteredOn(SourceTableState.class::isInstance) + .singleElement() + .satisfies(state -> assertThat(((SourceTableState) state).name()).isEqualTo(tableName)); + + assertThat(materializedViewIsFresh(materializedViewName)) + .as("the parent should be fresh after refresh") + .isTrue(); + + // Re-materializing a nested materialized view changes its storage table but not its version, + // so the parent -- which tracks it as a view -- stays fresh. + sql("REFRESH MATERIALIZED VIEW %s", gMv); + assertThat(materializedViewIsFresh(materializedViewName)) + .as("re-materializing a nested materialized view must not affect the parent's freshness") + .isTrue(); + + // Changing the base table the chain resolves to makes the parent stale, proving the deep base + // table snapshot was recorded and is being checked. + sql("INSERT INTO %s VALUES (3, 'c')", tableName); + assertThat(materializedViewIsFresh(materializedViewName)) + .as("a base-table change must make the parent stale") + .isFalse(); + + sql("DROP VIEW IF EXISTS %s", materializedViewName); + sql("DROP VIEW IF EXISTS %s", cMv); + sql("DROP VIEW IF EXISTS %s", gMv); + } + + /** + * A single refresh mixes both nested-materialized-view treatments: an upstream materialized view + * that is stale is expanded to its base table, while one that is fresh is read from its storage + * table and contributes its own recorded deep state through a merge. Both paths run in the same + * refresh, and the recorded state and the materialized data are correct. + */ + @TestTemplate + public void testRefreshMixesStaleExpansionAndFreshMerge() { + String staleMv = "stale_mv"; + String freshMv = "fresh_mv"; + String freshBase = "fresh_base"; + + // Stale branch reads `tableName`; fresh branch reads `fresh_base`. + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE TABLE %s (id INT, data STRING)", freshBase); + sql("INSERT INTO %s VALUES (1, 'x'), (2, 'y')", freshBase); + + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", staleMv, tableName); + sql("REFRESH MATERIALIZED VIEW %s", staleMv); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", freshMv, freshBase); + sql("REFRESH MATERIALIZED VIEW %s", freshMv); + + // Make only the stale branch stale: its base changes, the fresh branch's base does not. + sql("INSERT INTO %s VALUES (3, 'c')", tableName); + assertThat(materializedViewIsFresh(staleMv)).as("stale_mv should be stale").isFalse(); + assertThat(materializedViewIsFresh(freshMv)).as("fresh_mv should stay fresh").isTrue(); + + sql( + "CREATE MATERIALIZED VIEW %s AS " + + "SELECT s.id AS id, s.data AS data FROM %s s JOIN %s f ON s.id = f.id", + materializedViewName, staleMv, freshMv); + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + RefreshState refreshState = loadRefreshState(); + + // Both upstreams are recorded as views. + java.util.Set viewNames = + refreshState.sourceStates().stream() + .filter(SourceViewState.class::isInstance) + .map(state -> ((SourceViewState) state).name()) + .collect(java.util.stream.Collectors.toSet()); + assertThat(viewNames).containsExactlyInAnyOrder(staleMv, freshMv); + + // The stale upstream contributes its base table by expansion; the fresh upstream contributes + // its base table by merging its own recorded refresh state. + java.util.Set tableNames = + refreshState.sourceStates().stream() + .filter(SourceTableState.class::isInstance) + .map(state -> ((SourceTableState) state).name()) + .collect(java.util.stream.Collectors.toSet()); + assertThat(tableNames).containsExactlyInAnyOrder(tableName, freshBase); + + // Data is correct: stale_mv expanded to the current tableName {1,2,3}, fresh_mv served {1,2} + // from its storage table, so the join on id yields {1,2}. + assertThat(sql("SELECT id, data FROM %s ORDER BY id", materializedViewName)) + .containsExactly(row(1, "a"), row(2, "b")); + + sql("DROP VIEW IF EXISTS %s", materializedViewName); + sql("DROP VIEW IF EXISTS %s", staleMv); + sql("DROP VIEW IF EXISTS %s", freshMv); + sql("DROP TABLE IF EXISTS %s", freshBase); + } + + /** + * A base table referenced both directly and through a fresh nested materialized view is recorded + * once. The direct reference is recorded by the leaf pass, and the merge of the nested + * materialized view's refresh state deduplicates against it by uuid. + */ + @TestTemplate + public void testSharedBaseTableAcrossDirectAndNestedIsRecordedOnce() { + String childMv = "shared_child_mv"; + + sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b')", tableName); + sql("CREATE MATERIALIZED VIEW %s AS SELECT id, data FROM %s", childMv, tableName); + sql("REFRESH MATERIALIZED VIEW %s", childMv); + + // The parent reads the base table directly and also reads the fresh nested materialized view, + // whose own refresh state records that same base table. + sql( + "CREATE MATERIALIZED VIEW %s AS " + + "SELECT t.id AS id, t.data AS data FROM %s t JOIN %s c ON t.id = c.id", + materializedViewName, tableName, childMv); + sql("REFRESH MATERIALIZED VIEW %s", materializedViewName); + + RefreshState refreshState = loadRefreshState(); + + // Exactly two source states: the shared base table once and the nested materialized view as a + // view. The base table is not recorded twice despite the two reference paths. + assertThat(refreshState.sourceStates()).hasSize(2); + assertThat(refreshState.sourceStates()) + .filteredOn(SourceTableState.class::isInstance) + .singleElement() + .satisfies(state -> assertThat(((SourceTableState) state).name()).isEqualTo(tableName)); + assertThat(refreshState.sourceStates()) + .filteredOn(SourceViewState.class::isInstance) + .singleElement() + .satisfies(state -> assertThat(((SourceViewState) state).name()).isEqualTo(childMv)); + + sql("DROP VIEW IF EXISTS %s", materializedViewName); + sql("DROP VIEW IF EXISTS %s", childMv); + } + @TestTemplate public void testStaleWhenNestedViewChanges() { String sourceViewName = "source_view";