Repository navigation
[BUG] Nested arrays (ARRAY<ARRAY<INNER>>) cannot transport null values. #1658
Description
Describe the bug
select array(null)::ARRAY<ARRAY<VARCHAR(1)>>is valid and returns [null].
Such a result however cannot be retrieved via the JDBC Driver with getArray or getObject on the Databricks ResultSet implementation, instead it fails with Exception in thread "main" com.databricks.jdbc.exception.DatabricksDriverException: Error converting elements caused by a NullPointerException in DatabricksArray.
Sadly, there is no workaround, as all flows retrieving such an object ends up there.
To Reproduce
curl -O https://repo.maven.apache.org/maven2/com/databricks/databricks-jdbc/3.4.2/databricks-jdbc-3.4.2.jar
cat <<- 'EOF' > Application.java
import java.lang.reflect.Array;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;
import com.databricks.jdbc.common.DatabricksJdbcUrlParams;
public class Application {
public static void main(String... args) throws SQLException {
var url = "jdbc:databricks://" + System.getenv("HOST");
var catalog = System.getenv("CATALOG");
var httpPath = System.getenv("HTTP_PATH");
var token = System.getenv("TOKEN");
var properties = new Properties();
properties.setProperty(DatabricksJdbcUrlParams.CONN_CATALOG.getParamName(), catalog);
properties.setProperty(DatabricksJdbcUrlParams.HTTP_PATH.getParamName(), httpPath);
properties.setProperty(DatabricksJdbcUrlParams.SSL.getParamName(), "1");
properties.setProperty(DatabricksJdbcUrlParams.ENABLE_COMPLEX_DATATYPE_SUPPORT.getParamName(), "1");
properties.setProperty(DatabricksJdbcUrlParams.AUTH_MECH.getParamName(), "3");
properties.setProperty(DatabricksJdbcUrlParams.UID.getParamName(), "token");
properties.setProperty(DatabricksJdbcUrlParams.PWD.getParamName(), token);
try (var con = DriverManager.getConnection(url, properties); var stmt = con.createStatement()) {
var rs = stmt.executeQuery("SELECT array(null)::ARRAY<VARCHAR(32)> AS a");
rs.next();
var array = rs.getArray(1).getArray();
assert Array.get(array, 0) == null : "Array element expected to be null but was not";
rs.close();
rs = stmt.executeQuery("SELECT array(null)::ARRAY<ARRAY<VARCHAR(32)>> AS a");
rs.next();
array = rs.getArray(1).getArray(); // getArray(1) throws
assert Array.get(array, 0) == null : "Array element expected to be null but was not";
}
}
}
EOF
CATALOG=XXX \
HOST=xxx.cloud.databricks.com \
HTTP_PATH=/sql/1.0/warehouses/xxx \
TOKEN=dapiXXX \
java -cp databricks-jdbc-3.4.2.jar Application.java Expected behavior
Any null element of a nested array should be retrievable as Java null element the same way elements of a non-nested array are (see reproducer)
Screenshots
Not a screenshot, but the stack trace:
Exception in thread "main" com.databricks.jdbc.exception.DatabricksDriverException: Error converting elements
at com.databricks.jdbc.api.impl.DatabricksArray.convertElements(DatabricksArray.java:89)
at com.databricks.jdbc.api.impl.DatabricksArray.<init>(DatabricksArray.java:34)
at com.databricks.jdbc.api.impl.ComplexDataTypeParser.parseToArray(ComplexDataTypeParser.java:84)
at com.databricks.jdbc.api.impl.ComplexDataTypeParser.parseJsonStringToDbArray(ComplexDataTypeParser.java:40)
at com.databricks.jdbc.api.impl.converters.ArrowToJavaObjectConverter.convertToArray(ArrowToJavaObjectConverter.java:167)
at com.databricks.jdbc.api.impl.converters.ArrowToJavaObjectConverter.convert(ArrowToJavaObjectConverter.java:121)
at com.databricks.jdbc.api.impl.arrow.ArrowResultChunkIterator.getColumnObjectAtCurrentRow(ArrowResultChunkIterator.java:84)
at com.databricks.jdbc.api.impl.arrow.ArrowStreamResult.getObjectWithComplexTypeHandling(ArrowStreamResult.java:405)
at com.databricks.jdbc.api.impl.arrow.StreamingInlineArrowResult.getObject(StreamingInlineArrowResult.java:128)
at com.databricks.jdbc.api.impl.DatabricksResultSet.getObjectInternal(DatabricksResultSet.java:2327)
at com.databricks.jdbc.api.impl.DatabricksResultSet.getArray(DatabricksResultSet.java:1600)
at Application.main(Application.java:36)
Caused by: java.lang.NullPointerException: Cannot invoke "Object.getClass()" because "element" is null
at com.databricks.jdbc.api.impl.DatabricksArray.convertElements(DatabricksArray.java:69)
at com.databricks.jdbc.api.impl.DatabricksArray.<init>(DatabricksArray.java:34)
at com.databricks.jdbc.api.impl.ComplexDataTypeParser.parseToArray(ComplexDataTypeParser.java:84)
at com.databricks.jdbc.api.impl.ComplexDataTypeParser.parseJsonStringToDbArray(ComplexDataTypeParser.java:40)
at com.databricks.jdbc.api.impl.converters.ArrowToJavaObjectConverter.convertToArray(ArrowToJavaObjectConverter.java:167)
at com.databricks.jdbc.api.impl.converters.ArrowToJavaObjectConverter.convert(ArrowToJavaObjectConverter.java:121)
at com.databricks.jdbc.api.impl.arrow.ArrowResultChunkIterator.getColumnObjectAtCurrentRow(ArrowResultChunkIterator.java:84)
at com.databricks.jdbc.api.impl.arrow.ArrowStreamResult.getObjectWithComplexTypeHandling(ArrowStreamResult.java:405)
at com.databricks.jdbc.api.impl.arrow.StreamingInlineArrowResult.getObject(StreamingInlineArrowResult.java:128)
at com.databricks.jdbc.api.impl.DatabricksResultSet.getObjectInternal(DatabricksResultSet.java:2327)
at com.databricks.jdbc.api.impl.DatabricksResultSet.getArray(DatabricksResultSet.java:1600)
at Application.main(Application.java:36)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at jdk.compiler/com.sun.tools.javac.launcher.Main.execute(Main.java:484)
at jdk.compiler/com.sun.tools.javac.launcher.Main.run(Main.java:208)
at jdk.compiler/com.sun.tools.javac.launcher.Main.main(Main.java:135)
Client side logs
n/a
Client Environment (please complete the following information):
- OS: All but tested on MacOS 26.6.1
- Java version:
openjdk version "21" 2023-09-19
OpenJDK Runtime Environment Zulu21.28+85-CA (build 21+35)
OpenJDK 64-Bit Server VM Zulu21.28+85-CA (build 21+35, mixed mode, sharing) - Driver Version 3.4.2
Activity
Fix NPE on materialising nested arrays or arrays of maps with `NULL` …
Fix NPE on materialising nested arrays or arrays of maps or structs w…
Fix NPE on materialising nested arrays or arrays of maps with `NULL` …