diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index b69277e5a9..e0936d64e5 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -91,6 +91,7 @@ import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor; +import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; @@ -531,7 +532,7 @@ public Database getDatabase(String catName, String dbName) throws MetaException{ * @return List of partitions. */ public List getPartitionsViaSqlFilter(String catName, String dbName, String tableName, - SqlFilterForPushdown filter, Integer max) throws MetaException { + SqlFilterForPushdown filter, Integer max, Map params) throws MetaException { List partitionIds = getPartitionIdsViaSqlFilter(catName, dbName, tableName, filter.filter, filter.params, filter.joins, max); @@ -541,8 +542,9 @@ public Database getDatabase(String catName, String dbName) throws MetaException{ return Batchable.runBatched(batchSize, partitionIds, new Batchable() { @Override public List run(List input) throws MetaException { + boolean isTxnTable = TxnUtils.isTransactionalTable(params); return getPartitionsFromPartitionIds(catName, dbName, - tableName, null, input, Collections.emptyList()); + tableName, null, input, Collections.emptyList(), isTxnTable); } }); } @@ -776,6 +778,13 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw /** Should be called with the list short enough to not trip up Oracle/etc. */ private List getPartitionsFromPartitionIds(String catName, String dbName, String tblName, Boolean isView, List partIdList, List projectionFields) throws MetaException { + return getPartitionsFromPartitionIds(catName, dbName, tblName, isView, partIdList, projectionFields, false); + } + + /** Should be called with the list short enough to not trip up Oracle/etc. */ + private List getPartitionsFromPartitionIds(String catName, String dbName, String tblName, + Boolean isView, List partIdList, List projectionFields, + boolean isTxnTable) throws MetaException { boolean doTrace = LOG.isDebugEnabled(); @@ -914,13 +923,15 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw // Get all the stuff for SD. Don't do empty-list check - we expect partitions do have SDs. MetastoreDirectSqlUtils.setSDParameters(SD_PARAMS, convertMapNullsToEmptyStrings, pm, sds, sdIds); - MetastoreDirectSqlUtils.setSDSortCols(SORT_COLS, pm, sds, sdIds); + boolean hasSkewedColumns = false; + if (!isTxnTable) { + MetastoreDirectSqlUtils.setSDSortCols(SORT_COLS, pm, sds, sdIds); - MetastoreDirectSqlUtils.setSDBucketCols(BUCKETING_COLS, pm, sds, sdIds); + MetastoreDirectSqlUtils.setSDBucketCols(BUCKETING_COLS, pm, sds, sdIds); - // Skewed columns stuff. - boolean hasSkewedColumns = MetastoreDirectSqlUtils - .setSkewedColNames(SKEWED_COL_NAMES, pm, sds, sdIds); + // Skewed columns stuff. + hasSkewedColumns = MetastoreDirectSqlUtils.setSkewedColNames(SKEWED_COL_NAMES, pm, sds, sdIds); + } // Assume we don't need to fetch the rest of the skewed column data if we have no columns. if (hasSkewedColumns) { @@ -940,8 +951,9 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw } // Finally, get all the stuff for serdes - just the params. - MetastoreDirectSqlUtils - .setSerdeParams(SERDE_PARAMS, convertMapNullsToEmptyStrings, pm, serdes, serdeIds); + if (!isTxnTable) { + MetastoreDirectSqlUtils.setSerdeParams(SERDE_PARAMS, convertMapNullsToEmptyStrings, pm, serdes, serdeIds); + } return orderedResult; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 57488415ed..83de27548a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1939,10 +1939,12 @@ private Table convertToTable(MTable mtbl) throws MetaException { tableType = TableType.MANAGED_TABLE.toString(); } } + Map parameters = convertMap(mtbl.getParameters()); + boolean isTxnTable = TxnUtils.isTransactionalTable(parameters); final Table t = new Table(mtbl.getTableName(), mtbl.getDatabase().getName(), mtbl .getOwner(), mtbl.getCreateTime(), mtbl.getLastAccessTime(), mtbl - .getRetention(), convertToStorageDescriptor(mtbl.getSd()), - convertToFieldSchemas(mtbl.getPartitionKeys()), convertMap(mtbl.getParameters()), + .getRetention(), convertToStorageDescriptor(mtbl.getSd(), false, parameters, isTxnTable), + convertToFieldSchemas(mtbl.getPartitionKeys()), parameters, mtbl.getViewOriginalText(), mtbl.getViewExpandedText(), tableType); if (Strings.isNullOrEmpty(mtbl.getOwnerType())) { @@ -2118,6 +2120,31 @@ private StorageDescriptor convertToStorageDescriptor( return sd; } + private StorageDescriptor convertToStorageDescriptor( + MStorageDescriptor msd, boolean noFS, + Map parameters, boolean isTxnTable) throws MetaException { + if (msd == null) { + return null; + } + List mFieldSchemas = msd.getCD() == null ? null : msd.getCD().getCols(); + + List orderList = (isTxnTable) ? Collections.EMPTY_LIST : convertToOrders(msd.getSortCols()); + List bucList = (isTxnTable) ? Collections.EMPTY_LIST : convertList(msd.getBucketCols()); + + StorageDescriptor sd = new StorageDescriptor(noFS ? null : convertToFieldSchemas(mFieldSchemas), + msd.getLocation(), msd.getInputFormat(), msd.getOutputFormat(), msd + .isCompressed(), msd.getNumBuckets(), convertToSerDeInfo(msd.getSerDeInfo(), true), + bucList , orderList, parameters); + if (!isTxnTable) { + SkewedInfo skewedInfo = new SkewedInfo(convertList(msd.getSkewedColNames()), + convertToSkewedValues(msd.getSkewedColValues()), + covertToSkewedMap(msd.getSkewedColValueLocationMaps())); + sd.setSkewedInfo(skewedInfo); + } + sd.setStoredAsSubDirectories(msd.isStoredAsSubDirectories()); + return sd; + } + private StorageDescriptor convertToStorageDescriptor(MStorageDescriptor msd) throws MetaException { return convertToStorageDescriptor(msd, false); @@ -3470,6 +3497,7 @@ protected boolean getPartitionsByExprInternal(String catName, String dbName, Str MTable mTable = ensureGetMTable(catName, dbName, tblName); List partitionKeys = convertToFieldSchemas(mTable.getPartitionKeys()); + Map parameters = mTable.getParameters(); result.addAll(new GetListHelper(catName, dbName, tblName, allowSql, allowJdo) { @Override protected List getSqlResult(GetHelper> ctx) throws MetaException { @@ -3479,7 +3507,7 @@ protected boolean getPartitionsByExprInternal(String catName, String dbName, Str if (directSql.generateSqlFilterForPushdown(catName, dbName, tblName, partitionKeys, exprTree, defaultPartitionName, filter)) { String catalogName = (catName != null) ? catName : DEFAULT_CATALOG_NAME; - return directSql.getPartitionsViaSqlFilter(catalogName, dbName, tblName, filter, null); + return directSql.getPartitionsViaSqlFilter(catalogName, dbName, tblName, filter, null, parameters); } } // We couldn't do SQL filter pushdown. Get names via normal means. @@ -3979,7 +4007,7 @@ public int getNumPartitionsByFilter(String catName, String dbName, String tblNam tblName = normalizeIdentifier(tblName); MTable mTable = ensureGetMTable(catName, dbName, tblName); List partitionKeys = convertToFieldSchemas(mTable.getPartitionKeys()); - + Map parameters = mTable.getParameters(); return new GetHelper(catName, dbName, tblName, true, true) { private final SqlFilterForPushdown filter = new SqlFilterForPushdown(); @@ -4070,6 +4098,7 @@ protected Integer getJdoResult( MTable mTable = ensureGetMTable(catName, dbName, tblName); List partitionKeys = convertToFieldSchemas(mTable.getPartitionKeys()); + Map parameters = mTable.getParameters(); final ExpressionTree tree = (filter != null && !filter.isEmpty()) ? PartFilterExprUtil.getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE; return new GetListHelper(catName, dbName, tblName, allowSql, allowJdo) { @@ -4082,7 +4111,7 @@ protected boolean canUseDirectSql(GetHelper> ctx) throws MetaExc @Override protected List getSqlResult(GetHelper> ctx) throws MetaException { - return directSql.getPartitionsViaSqlFilter(catName, dbName, tblName, filter, (maxParts < 0) ? null : (int)maxParts); + return directSql.getPartitionsViaSqlFilter(catName, dbName, tblName, filter, (maxParts < 0) ? null : (int)maxParts, parameters); } @Override