diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 90e9b2b7094b..8030708f9216 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1993,7 +1993,8 @@ public static enum ConfVars { "org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe," + "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe," + "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe," + - "org.apache.hadoop.hive.serde2.OpenCSVSerde", + "org.apache.hadoop.hive.serde2.OpenCSVSerde," + + "org.apache.hadoop.hive.hbase.HBaseSerDe", "SerDes retrieving schema from metastore. This is an internal parameter."), @Deprecated diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java index fadd793bc23a..1385141a3500 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java @@ -113,9 +113,9 @@ private void checkFields(List fieldSchemas, List field @Test public void testAvroTableWithDefaultSSR() throws Exception { hiveConf.set("metastore.storage.schema.reader.impl", "org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader"); String tblName = "avroTable"; - createTable(tblName, AvroSerDe.class.getName(), AvroContainerInputFormat.class.getName(), + Table tbl = createTable(tblName, AvroSerDe.class.getName(), AvroContainerInputFormat.class.getName(), AvroContainerOutputFormat.class.getName(), avroTableParams, new HashMap<>()); - assertThrows("Storage schema reading not supported", MetaException.class, () -> client.getSchema(dbName, tblName)); + checkSchema(tblName, tbl); } @Test public void testAvroTableWithSerdeSSR() throws Exception { @@ -128,15 +128,15 @@ private void checkFields(List fieldSchemas, List field @Test public void testHbaseTableWithDefaultSSR() throws Exception { hiveConf.set("metastore.storage.schema.reader.impl", "org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader"); - String tblName = "jdbcTable"; + String tblName = "hbaseTable"; - createTable(tblName, HBaseSerDe.class.getName(), null, null, hbaseTableParams, hbaseSerdeParams); - assertThrows("Storage schema reading not supported", MetaException.class, () -> client.getSchema(dbName, tblName)); + Table table = createTable(tblName, HBaseSerDe.class.getName(), null, null, hbaseTableParams, hbaseSerdeParams); + checkSchema(tblName, table); } @Test public void testHbaseTableWithSerdeSSR() throws Exception { hiveConf.set("metastore.storage.schema.reader.impl", "org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader"); - String tblName = "jdbcTable"; + String tblName = "hbaseTableSerde"; Table table = createTable(tblName, "org.apache.hadoop.hive.hbase.HBaseSerDe", null, null, hbaseTableParams, hbaseSerdeParams); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 16ccf508a43d..10015f74837c 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1441,7 +1441,8 @@ public enum ConfVars { "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe," + "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe," + "org.apache.hadoop.hive.serde2.OpenCSVSerde," + - "org.apache.iceberg.mr.hive.HiveIcebergSerDe", + "org.apache.iceberg.mr.hive.HiveIcebergSerDe," + + "org.apache.hadoop.hive.hbase.HBaseSerDe", "SerDes retrieving schema from metastore. This is an internal parameter."), SERDES_WITHOUT_FROM_DESERIALIZER("metastore.serdes.without.from.deserializer", "hive.metastore.serdes.without.from.deserializer", diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java index 6065574d9690..3e3bb7f596fc 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java @@ -4271,14 +4271,25 @@ private List get_fields_with_environment_context_core(String db, St } catch (NoSuchObjectException e) { throw new UnknownTableException(e.getMessage()); } - if (null == tbl.getSd().getSerdeInfo().getSerializationLib() || + String serdeLib = tbl.getSd().getSerdeInfo().getSerializationLib(); + if (serdeLib == null || MetastoreConf.getStringCollection(conf, - ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA).contains( - tbl.getSd().getSerdeInfo().getSerializationLib())) { + ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA).contains(serdeLib)) { ret = tbl.getSd().getCols(); } else { - StorageSchemaReader schemaReader = getStorageSchemaReader(); - ret = schemaReader.readSchema(tbl, envContext, getConf()); + try { + StorageSchemaReader schemaReader = getStorageSchemaReader(); + ret = schemaReader.readSchema(tbl, envContext, getConf()); + } catch (Exception e) { + if ("org.apache.hadoop.hive.serde2.avro.AvroSerDe".equals(serdeLib)) { + LOG.warn("Unable to read schema from storage for AvroSerDe table '{}.{}' ({}). " + + "Returning metastore SD columns as fallback; schema may be stale ", + db, tableName, e.getMessage()); + ret = tbl.getSd().getCols(); + } else { + throw new UnsupportedOperationException("Storage schema reading not supported"); + } + } } } catch (Exception e) { ex = e; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 9e52ca13b1fa..048cb58978a6 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -3958,4 +3958,72 @@ public void testDropDataConnectorIfNotExistsTrue() throws Exception { // No such data connector, ignore NoSuchObjectException client.dropDataConnector("no_such_data_connector", true, false); } + + @Test + public void testGetFieldsForAvroSerDe() throws Exception { + String dbName = "test_avro_serde_db"; + String avroTbl = "avro_tbl"; + + client.dropTable(dbName, avroTbl, true, true); + silentDropDatabase(dbName); + + new DatabaseBuilder() + .setName(dbName) + .create(client, conf); + + new TableBuilder() + .setDbName(dbName) + .setTableName(avroTbl) + .setSerdeLib("org.apache.hadoop.hive.serde2.avro.AvroSerDe") + .setInputFormat("org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat") + .setOutputFormat("org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat") + .addCol("foo", "int", "") + .addCol("bar", "string", "") + .addCol("baz", "bigint", "") + .create(client, conf); + + List fields = client.getFields(dbName, avroTbl); + assertEquals("AvroSerDe table should return 3 fields from metastore", 3, fields.size()); + assertEquals("foo", fields.get(0).getName()); + assertEquals("bar", fields.get(1).getName()); + assertEquals("baz", fields.get(2).getName()); + + List schema = client.getSchema(dbName, avroTbl); + assertEquals("AvroSerDe getSchema should return 3 columns", 3, schema.size()); + + client.dropTable(dbName, avroTbl, true, true); + client.dropDatabase(dbName); + } + + @Test + public void testGetFieldsForHBaseSerDe() throws Exception { + String dbName = "test_hbase_serde_db"; + String hbaseTbl = "hbase_tbl"; + + client.dropTable(dbName, hbaseTbl, true, true); + silentDropDatabase(dbName); + + new DatabaseBuilder() + .setName(dbName) + .create(client, conf); + + new TableBuilder() + .setDbName(dbName) + .setTableName(hbaseTbl) + .setSerdeLib("org.apache.hadoop.hive.hbase.HBaseSerDe") + .addCol("key", "string", "") + .addCol("value", "string", "") + .create(client, conf); + + List fields = client.getFields(dbName, hbaseTbl); + assertEquals("HBaseSerDe table should return 2 fields from metastore", 2, fields.size()); + assertEquals("key", fields.get(0).getName()); + assertEquals("value", fields.get(1).getName()); + + List schema = client.getSchema(dbName, hbaseTbl); + assertEquals("HBaseSerDe getSchema should return 2 columns", 2, schema.size()); + + client.dropTable(dbName, hbaseTbl, true, true); + client.dropDatabase(dbName); + } }