Skip to content

Commit 170e8e5

Browse files
deniskuzZdeniskuzZ
authored
HIVE-29773: Iceberg: Support CAST(x AS VARIANT) (#6654)
1 parent f443719 commit 170e8e5

7 files changed

Lines changed: 524 additions & 1 deletion

File tree

‎iceberg/iceberg-handler/src/test/queries/positive/variant_cast.q‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
set hive.explain.user=false;
2+
3+
drop table if exists variant_cast;
4+
5+
CREATE EXTERNAL TABLE variant_cast (
6+
id INT,
7+
v VARIANT
8+
) STORED BY ICEBERG
9+
TBLPROPERTIES (
10+
'format-version'='3'
11+
);
12+
13+
-- typed variant construction: values JSON cannot express keep their real type
14+
INSERT INTO variant_cast VALUES
15+
(1, CAST(true AS VARIANT)),
16+
(2, CAST(42 AS VARIANT)),
17+
(3, CAST(1.5BD AS VARIANT)),
18+
(4, CAST(unhex('DEADBEEF') AS VARIANT)),
19+
(5, CAST(DATE '2026-07-28' AS VARIANT)),
20+
(6, CAST(TIMESTAMP '2026-07-28 10:00:00' AS VARIANT)),
21+
(7, CAST('plain' AS VARIANT)),
22+
(8, CAST(array(1, 2, 3) AS VARIANT)),
23+
(9, CAST(NULL AS VARIANT));
24+
25+
SELECT id, v FROM variant_cast ORDER BY id;
26+
27+
-- binary node is rendered as base64, matching Impala/Trino
28+
SELECT variant_get(v, '$', 'string') FROM variant_cast WHERE id = 4;
29+
30+
-- cast over a column expression, not a constant
31+
drop table if exists variant_src;
32+
33+
CREATE TABLE variant_src (id INT, s STRING);
34+
35+
INSERT INTO variant_src VALUES (11, 'from_col');
36+
37+
EXPLAIN INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src;
38+
39+
INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src;
40+
41+
SELECT id, v FROM variant_cast WHERE id > 10;
42+
43+
-- identity cast over a reader-produced variant column
44+
SELECT id, CAST(v AS VARIANT) FROM variant_cast WHERE id > 10;
45+
46+
drop table variant_cast;
47+
drop table variant_src;

‎iceberg/iceberg-handler/src/test/results/positive/variant_cast.q.out‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
PREHOOK: query: drop table if exists variant_cast
2+
PREHOOK: type: DROPTABLE
3+
PREHOOK: Output: database:default
4+
POSTHOOK: query: drop table if exists variant_cast
5+
POSTHOOK: type: DROPTABLE
6+
POSTHOOK: Output: database:default
7+
PREHOOK: query: CREATE EXTERNAL TABLE variant_cast (
8+
id INT,
9+
v VARIANT
10+
) STORED BY ICEBERG
11+
TBLPROPERTIES (
12+
'format-version'='3'
13+
)
14+
PREHOOK: type: CREATETABLE
15+
PREHOOK: Output: database:default
16+
PREHOOK: Output: default@variant_cast
17+
POSTHOOK: query: CREATE EXTERNAL TABLE variant_cast (
18+
id INT,
19+
v VARIANT
20+
) STORED BY ICEBERG
21+
TBLPROPERTIES (
22+
'format-version'='3'
23+
)
24+
POSTHOOK: type: CREATETABLE
25+
POSTHOOK: Output: database:default
26+
POSTHOOK: Output: default@variant_cast
27+
PREHOOK: query: INSERT INTO variant_cast VALUES
28+
(1, CAST(true AS VARIANT)),
29+
(2, CAST(42 AS VARIANT)),
30+
(3, CAST(1.5BD AS VARIANT)),
31+
(4, CAST(unhex('DEADBEEF') AS VARIANT)),
32+
(5, CAST(DATE '2026-07-28' AS VARIANT)),
33+
(6, CAST(TIMESTAMP '2026-07-28 10:00:00' AS VARIANT)),
34+
(7, CAST('plain' AS VARIANT)),
35+
(8, CAST(array(1, 2, 3) AS VARIANT)),
36+
(9, CAST(NULL AS VARIANT))
37+
PREHOOK: type: QUERY
38+
PREHOOK: Input: _dummy_database@_dummy_table
39+
PREHOOK: Output: default@variant_cast
40+
POSTHOOK: query: INSERT INTO variant_cast VALUES
41+
(1, CAST(true AS VARIANT)),
42+
(2, CAST(42 AS VARIANT)),
43+
(3, CAST(1.5BD AS VARIANT)),
44+
(4, CAST(unhex('DEADBEEF') AS VARIANT)),
45+
(5, CAST(DATE '2026-07-28' AS VARIANT)),
46+
(6, CAST(TIMESTAMP '2026-07-28 10:00:00' AS VARIANT)),
47+
(7, CAST('plain' AS VARIANT)),
48+
(8, CAST(array(1, 2, 3) AS VARIANT)),
49+
(9, CAST(NULL AS VARIANT))
50+
POSTHOOK: type: QUERY
51+
POSTHOOK: Input: _dummy_database@_dummy_table
52+
POSTHOOK: Output: default@variant_cast
53+
PREHOOK: query: SELECT id, v FROM variant_cast ORDER BY id
54+
PREHOOK: type: QUERY
55+
PREHOOK: Input: default@variant_cast
56+
PREHOOK: Output: hdfs://### HDFS PATH ###
57+
POSTHOOK: query: SELECT id, v FROM variant_cast ORDER BY id
58+
POSTHOOK: type: QUERY
59+
POSTHOOK: Input: default@variant_cast
60+
POSTHOOK: Output: hdfs://### HDFS PATH ###
61+
1 true
62+
2 42
63+
3 1.5
64+
4 "3q2+7w=="
65+
5 "2026-07-28"
66+
6 "2026-07-28 10:00:00"
67+
7 "plain"
68+
8 [1,2,3]
69+
9 NULL
70+
PREHOOK: query: SELECT variant_get(v, '$', 'string') FROM variant_cast WHERE id = 4
71+
PREHOOK: type: QUERY
72+
PREHOOK: Input: default@variant_cast
73+
PREHOOK: Output: hdfs://### HDFS PATH ###
74+
POSTHOOK: query: SELECT variant_get(v, '$', 'string') FROM variant_cast WHERE id = 4
75+
POSTHOOK: type: QUERY
76+
POSTHOOK: Input: default@variant_cast
77+
POSTHOOK: Output: hdfs://### HDFS PATH ###
78+
3q2+7w==
79+
PREHOOK: query: drop table if exists variant_src
80+
PREHOOK: type: DROPTABLE
81+
PREHOOK: Output: database:default
82+
POSTHOOK: query: drop table if exists variant_src
83+
POSTHOOK: type: DROPTABLE
84+
POSTHOOK: Output: database:default
85+
PREHOOK: query: CREATE TABLE variant_src (id INT, s STRING)
86+
PREHOOK: type: CREATETABLE
87+
PREHOOK: Output: database:default
88+
PREHOOK: Output: default@variant_src
89+
POSTHOOK: query: CREATE TABLE variant_src (id INT, s STRING)
90+
POSTHOOK: type: CREATETABLE
91+
POSTHOOK: Output: database:default
92+
POSTHOOK: Output: default@variant_src
93+
PREHOOK: query: INSERT INTO variant_src VALUES (11, 'from_col')
94+
PREHOOK: type: QUERY
95+
PREHOOK: Input: _dummy_database@_dummy_table
96+
PREHOOK: Output: default@variant_src
97+
POSTHOOK: query: INSERT INTO variant_src VALUES (11, 'from_col')
98+
POSTHOOK: type: QUERY
99+
POSTHOOK: Input: _dummy_database@_dummy_table
100+
POSTHOOK: Output: default@variant_src
101+
POSTHOOK: Lineage: variant_src.id SCRIPT []
102+
POSTHOOK: Lineage: variant_src.s SCRIPT []
103+
PREHOOK: query: EXPLAIN INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src
104+
PREHOOK: type: QUERY
105+
PREHOOK: Input: default@variant_src
106+
PREHOOK: Output: default@variant_cast
107+
POSTHOOK: query: EXPLAIN INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src
108+
POSTHOOK: type: QUERY
109+
POSTHOOK: Input: default@variant_src
110+
POSTHOOK: Output: default@variant_cast
111+
STAGE DEPENDENCIES:
112+
Stage-1 is a root stage
113+
Stage-2 depends on stages: Stage-1
114+
Stage-0 depends on stages: Stage-2
115+
Stage-3 depends on stages: Stage-0
116+
117+
STAGE PLANS:
118+
Stage: Stage-1
119+
Tez
120+
#### A masked pattern was here ####
121+
Vertices:
122+
Map 1
123+
Map Operator Tree:
124+
TableScan
125+
alias: variant_src
126+
Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
127+
Select Operator
128+
expressions: id (type: int), CAST( s AS VARIANT) (type: struct<metadata:binary,value:binary>)
129+
outputColumnNames: _col0, _col1
130+
Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE
131+
File Output Operator
132+
compressed: false
133+
Statistics: Num rows: 1 Data size: 340 Basic stats: COMPLETE Column stats: COMPLETE
134+
table:
135+
input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
136+
output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
137+
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
138+
name: default.variant_cast
139+
Execution mode: vectorized
140+
141+
Stage: Stage-2
142+
Dependency Collection
143+
144+
Stage: Stage-0
145+
Move Operator
146+
tables:
147+
replace: false
148+
table:
149+
input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
150+
output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
151+
serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
152+
name: default.variant_cast
153+
154+
Stage: Stage-3
155+
Stats Work
156+
Basic Stats Work:
157+
158+
PREHOOK: query: INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src
159+
PREHOOK: type: QUERY
160+
PREHOOK: Input: default@variant_src
161+
PREHOOK: Output: default@variant_cast
162+
POSTHOOK: query: INSERT INTO variant_cast SELECT id, CAST(s AS VARIANT) FROM variant_src
163+
POSTHOOK: type: QUERY
164+
POSTHOOK: Input: default@variant_src
165+
POSTHOOK: Output: default@variant_cast
166+
PREHOOK: query: SELECT id, v FROM variant_cast WHERE id > 10
167+
PREHOOK: type: QUERY
168+
PREHOOK: Input: default@variant_cast
169+
PREHOOK: Output: hdfs://### HDFS PATH ###
170+
POSTHOOK: query: SELECT id, v FROM variant_cast WHERE id > 10
171+
POSTHOOK: type: QUERY
172+
POSTHOOK: Input: default@variant_cast
173+
POSTHOOK: Output: hdfs://### HDFS PATH ###
174+
11 "from_col"
175+
PREHOOK: query: SELECT id, CAST(v AS VARIANT) FROM variant_cast WHERE id > 10
176+
PREHOOK: type: QUERY
177+
PREHOOK: Input: default@variant_cast
178+
PREHOOK: Output: hdfs://### HDFS PATH ###
179+
POSTHOOK: query: SELECT id, CAST(v AS VARIANT) FROM variant_cast WHERE id > 10
180+
POSTHOOK: type: QUERY
181+
POSTHOOK: Input: default@variant_cast
182+
POSTHOOK: Output: hdfs://### HDFS PATH ###
183+
11 "from_col"
184+
PREHOOK: query: drop table variant_cast
185+
PREHOOK: type: DROPTABLE
186+
PREHOOK: Input: default@variant_cast
187+
PREHOOK: Output: database:default
188+
PREHOOK: Output: default@variant_cast
189+
POSTHOOK: query: drop table variant_cast
190+
POSTHOOK: type: DROPTABLE
191+
POSTHOOK: Input: default@variant_cast
192+
POSTHOOK: Output: database:default
193+
POSTHOOK: Output: default@variant_cast
194+
PREHOOK: query: drop table variant_src
195+
PREHOOK: type: DROPTABLE
196+
PREHOOK: Input: default@variant_src
197+
PREHOOK: Output: database:default
198+
PREHOOK: Output: default@variant_src
199+
POSTHOOK: query: drop table variant_src
200+
POSTHOOK: type: DROPTABLE
201+
POSTHOOK: Input: default@variant_src
202+
POSTHOOK: Output: database:default
203+
POSTHOOK: Output: default@variant_src

‎ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -789,6 +789,7 @@ public final class FunctionRegistry {
789789
system.registerFunction("ST_Z", ST_Z.class);
790790

791791
// Iceberg UDFs
792+
system.registerGenericUDF(serdeConstants.VARIANT_TYPE_NAME, GenericUDFToVariant.class);
792793
system.registerGenericUDF("parse_json", GenericUDFParseJson.class);
793794
system.registerGenericUDF("to_json", GenericUDFToJson.class);
794795
system.registerGenericUDF("variant_get", GenericUDFVariantGet.class);
@@ -1638,7 +1639,7 @@ public static boolean isOpCast(GenericUDF genericUDF) {
16381639
udfClass == GenericUDFToVarchar.class || udfClass == GenericUDFToChar.class ||
16391640
udfClass == GenericUDFTimestamp.class || udfClass == GenericUDFToBinary.class ||
16401641
udfClass == GenericUDFToDate.class || udfClass == GenericUDFToDecimal.class ||
1641-
udfClass == GenericUDFToTimestampLocalTZ.class;
1642+
udfClass == GenericUDFToTimestampLocalTZ.class || udfClass == GenericUDFToVariant.class;
16421643
}
16431644

16441645
/**

‎ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,8 @@ public class TypeCheckProcFactory<T> {
140140
serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME);
141141
CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_DECIMAL,
142142
serdeConstants.DECIMAL_TYPE_NAME);
143+
CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_VARIANT,
144+
serdeConstants.VARIANT_TYPE_NAME);
143145
CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_MAP, "toMap");
144146
CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_LIST, "toArray");
145147
CONVERSION_FUNCTION_TEXT_MAP.put(HiveParser.TOK_STRUCT, "toStruct");

0 commit comments

Comments
 (0)