From 193ee2558ac00a6ae1c98a5e110d64c9e20e0686 Mon Sep 17 00:00:00 2001 From: rtrivedi Date: Thu, 5 Mar 2026 23:32:20 -0600 Subject: [PATCH 1/2] HIVE-29484: HMS.getFields() fails for Avro and Hbase tables --- .../hive/metastore/conf/MetastoreConf.java | 4 +- .../hive/metastore/TestHiveMetaStore.java | 54 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 16ccf508a43d..bb0b02be7711 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1441,7 +1441,9 @@ public enum ConfVars { "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe," + "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe," + "org.apache.hadoop.hive.serde2.OpenCSVSerde," + - "org.apache.iceberg.mr.hive.HiveIcebergSerDe", + "org.apache.iceberg.mr.hive.HiveIcebergSerDe," + + "org.apache.hadoop.hive.serde2.avro.AvroSerDe," + + "org.apache.hadoop.hive.hbase.HBaseSerDe", "SerDes retrieving schema from metastore. This is an internal parameter."), SERDES_WITHOUT_FROM_DESERIALIZER("metastore.serdes.without.from.deserializer", "hive.metastore.serdes.without.from.deserializer", diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 9e52ca13b1fa..326c9de4567a 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -3958,4 +3958,58 @@ public void testDropDataConnectorIfNotExistsTrue() throws Exception { // No such data connector, ignore NoSuchObjectException client.dropDataConnector("no_such_data_connector", true, false); } + + /** + * Test for storage-based SerDes (AvroSerDe, HBaseSerDe).If the SerDe is not listed in + * SERDES_USING_METASTORE_FOR_SCHEMA, HMS falls through to DefaultStorageSchemaReader and throws + * "Storage schema reading not supported". + */ + @Test + public void testGetFieldsForStorageSerDes() throws Exception { + String dbName = "test_storage_serde_db"; + String avroTbl = "avro_tbl"; + String hbaseTbl = "hbase_tbl"; + + client.dropTable(dbName, avroTbl, true, true); + client.dropTable(dbName, hbaseTbl, true, true); + silentDropDatabase(dbName); + + new DatabaseBuilder() + .setName(dbName) + .create(client, conf); + + new TableBuilder() + .setDbName(dbName) + .setTableName(avroTbl) + .setSerdeLib("org.apache.hadoop.hive.serde2.avro.AvroSerDe") + .setInputFormat("org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat") + .setOutputFormat("org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat") + .addCol("foo", "int", "") + .addCol("bar", "string", "") + .addCol("baz", "bigint", "") + .create(client, conf); + + List avroFields = client.getFields(dbName, avroTbl); + assertEquals("AvroSerDe table should return 3 fields from metastore", 3, avroFields.size()); + assertEquals("foo", avroFields.get(0).getName()); + assertEquals("bar", avroFields.get(1).getName()); + assertEquals("baz", avroFields.get(2).getName()); + + new TableBuilder() + .setDbName(dbName) + .setTableName(hbaseTbl) + .setSerdeLib("org.apache.hadoop.hive.hbase.HBaseSerDe") + .addCol("key", "string", "") + .addCol("value", "string", "") + .create(client, conf); + + List hbaseFields = client.getFields(dbName, hbaseTbl); + assertEquals("HBaseSerDe table should return 2 fields from metastore", 2, hbaseFields.size()); + assertEquals("key", hbaseFields.get(0).getName()); + assertEquals("value", hbaseFields.get(1).getName()); + + client.dropTable(dbName, avroTbl, true, true); + client.dropTable(dbName, hbaseTbl, true, true); + client.dropDatabase(dbName); + } } From 4d670c19c253b43227594486c794c6fe4c97ce56 Mon Sep 17 00:00:00 2001 From: rtrivedi Date: Tue, 10 Mar 2026 23:55:10 -0500 Subject: [PATCH 2/2] HIVE-29484: Removed AVRO from SERDES_USING_METASTORE_FOR_SCHEMA to preserve Avro schema evolution in HS2 --- .../org/apache/hadoop/hive/conf/HiveConf.java | 3 +- .../metastore/TestStorageSchemaReader.java | 12 ++--- .../hive/metastore/conf/MetastoreConf.java | 1 - .../hadoop/hive/metastore/HMSHandler.java | 21 ++++++-- .../hive/metastore/TestHiveMetaStore.java | 54 ++++++++++++------- 5 files changed, 58 insertions(+), 33 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 90e9b2b7094b..8030708f9216 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1993,7 +1993,8 @@ public static enum ConfVars { "org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe," + "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe," + "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe," + - "org.apache.hadoop.hive.serde2.OpenCSVSerde", + "org.apache.hadoop.hive.serde2.OpenCSVSerde," + + "org.apache.hadoop.hive.hbase.HBaseSerDe", "SerDes retrieving schema from metastore. This is an internal parameter."), @Deprecated diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java index fadd793bc23a..1385141a3500 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java @@ -113,9 +113,9 @@ private void checkFields(List fieldSchemas, List field @Test public void testAvroTableWithDefaultSSR() throws Exception { hiveConf.set("metastore.storage.schema.reader.impl", "org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader"); String tblName = "avroTable"; - createTable(tblName, AvroSerDe.class.getName(), AvroContainerInputFormat.class.getName(), + Table tbl = createTable(tblName, AvroSerDe.class.getName(), AvroContainerInputFormat.class.getName(), AvroContainerOutputFormat.class.getName(), avroTableParams, new HashMap<>()); - assertThrows("Storage schema reading not supported", MetaException.class, () -> client.getSchema(dbName, tblName)); + checkSchema(tblName, tbl); } @Test public void testAvroTableWithSerdeSSR() throws Exception { @@ -128,15 +128,15 @@ private void checkFields(List fieldSchemas, List field @Test public void testHbaseTableWithDefaultSSR() throws Exception { hiveConf.set("metastore.storage.schema.reader.impl", "org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader"); - String tblName = "jdbcTable"; + String tblName = "hbaseTable"; - createTable(tblName, HBaseSerDe.class.getName(), null, null, hbaseTableParams, hbaseSerdeParams); - assertThrows("Storage schema reading not supported", MetaException.class, () -> client.getSchema(dbName, tblName)); + Table table = createTable(tblName, HBaseSerDe.class.getName(), null, null, hbaseTableParams, hbaseSerdeParams); + checkSchema(tblName, table); } @Test public void testHbaseTableWithSerdeSSR() throws Exception { hiveConf.set("metastore.storage.schema.reader.impl", "org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader"); - String tblName = "jdbcTable"; + String tblName = "hbaseTableSerde"; Table table = createTable(tblName, "org.apache.hadoop.hive.hbase.HBaseSerDe", null, null, hbaseTableParams, hbaseSerdeParams); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index bb0b02be7711..10015f74837c 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1442,7 +1442,6 @@ public enum ConfVars { "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe," + "org.apache.hadoop.hive.serde2.OpenCSVSerde," + "org.apache.iceberg.mr.hive.HiveIcebergSerDe," + - "org.apache.hadoop.hive.serde2.avro.AvroSerDe," + "org.apache.hadoop.hive.hbase.HBaseSerDe", "SerDes retrieving schema from metastore. This is an internal parameter."), SERDES_WITHOUT_FROM_DESERIALIZER("metastore.serdes.without.from.deserializer", diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java index 6065574d9690..3e3bb7f596fc 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java @@ -4271,14 +4271,25 @@ private List get_fields_with_environment_context_core(String db, St } catch (NoSuchObjectException e) { throw new UnknownTableException(e.getMessage()); } - if (null == tbl.getSd().getSerdeInfo().getSerializationLib() || + String serdeLib = tbl.getSd().getSerdeInfo().getSerializationLib(); + if (serdeLib == null || MetastoreConf.getStringCollection(conf, - ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA).contains( - tbl.getSd().getSerdeInfo().getSerializationLib())) { + ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA).contains(serdeLib)) { ret = tbl.getSd().getCols(); } else { - StorageSchemaReader schemaReader = getStorageSchemaReader(); - ret = schemaReader.readSchema(tbl, envContext, getConf()); + try { + StorageSchemaReader schemaReader = getStorageSchemaReader(); + ret = schemaReader.readSchema(tbl, envContext, getConf()); + } catch (Exception e) { + if ("org.apache.hadoop.hive.serde2.avro.AvroSerDe".equals(serdeLib)) { + LOG.warn("Unable to read schema from storage for AvroSerDe table '{}.{}' ({}). " + + "Returning metastore SD columns as fallback; schema may be stale ", + db, tableName, e.getMessage()); + ret = tbl.getSd().getCols(); + } else { + throw new UnsupportedOperationException("Storage schema reading not supported"); + } + } } } catch (Exception e) { ex = e; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 326c9de4567a..048cb58978a6 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -3958,20 +3958,13 @@ public void testDropDataConnectorIfNotExistsTrue() throws Exception { // No such data connector, ignore NoSuchObjectException client.dropDataConnector("no_such_data_connector", true, false); } - - /** - * Test for storage-based SerDes (AvroSerDe, HBaseSerDe).If the SerDe is not listed in - * SERDES_USING_METASTORE_FOR_SCHEMA, HMS falls through to DefaultStorageSchemaReader and throws - * "Storage schema reading not supported". - */ + @Test - public void testGetFieldsForStorageSerDes() throws Exception { - String dbName = "test_storage_serde_db"; + public void testGetFieldsForAvroSerDe() throws Exception { + String dbName = "test_avro_serde_db"; String avroTbl = "avro_tbl"; - String hbaseTbl = "hbase_tbl"; client.dropTable(dbName, avroTbl, true, true); - client.dropTable(dbName, hbaseTbl, true, true); silentDropDatabase(dbName); new DatabaseBuilder() @@ -3989,11 +3982,30 @@ public void testGetFieldsForStorageSerDes() throws Exception { .addCol("baz", "bigint", "") .create(client, conf); - List avroFields = client.getFields(dbName, avroTbl); - assertEquals("AvroSerDe table should return 3 fields from metastore", 3, avroFields.size()); - assertEquals("foo", avroFields.get(0).getName()); - assertEquals("bar", avroFields.get(1).getName()); - assertEquals("baz", avroFields.get(2).getName()); + List fields = client.getFields(dbName, avroTbl); + assertEquals("AvroSerDe table should return 3 fields from metastore", 3, fields.size()); + assertEquals("foo", fields.get(0).getName()); + assertEquals("bar", fields.get(1).getName()); + assertEquals("baz", fields.get(2).getName()); + + List schema = client.getSchema(dbName, avroTbl); + assertEquals("AvroSerDe getSchema should return 3 columns", 3, schema.size()); + + client.dropTable(dbName, avroTbl, true, true); + client.dropDatabase(dbName); + } + + @Test + public void testGetFieldsForHBaseSerDe() throws Exception { + String dbName = "test_hbase_serde_db"; + String hbaseTbl = "hbase_tbl"; + + client.dropTable(dbName, hbaseTbl, true, true); + silentDropDatabase(dbName); + + new DatabaseBuilder() + .setName(dbName) + .create(client, conf); new TableBuilder() .setDbName(dbName) @@ -4003,12 +4015,14 @@ public void testGetFieldsForStorageSerDes() throws Exception { .addCol("value", "string", "") .create(client, conf); - List hbaseFields = client.getFields(dbName, hbaseTbl); - assertEquals("HBaseSerDe table should return 2 fields from metastore", 2, hbaseFields.size()); - assertEquals("key", hbaseFields.get(0).getName()); - assertEquals("value", hbaseFields.get(1).getName()); + List fields = client.getFields(dbName, hbaseTbl); + assertEquals("HBaseSerDe table should return 2 fields from metastore", 2, fields.size()); + assertEquals("key", fields.get(0).getName()); + assertEquals("value", fields.get(1).getName()); + + List schema = client.getSchema(dbName, hbaseTbl); + assertEquals("HBaseSerDe getSchema should return 2 columns", 2, schema.size()); - client.dropTable(dbName, avroTbl, true, true); client.dropTable(dbName, hbaseTbl, true, true); client.dropDatabase(dbName); }