diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 1f89b7c..0e6e343 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -97,6 +97,7 @@ private static final Log LOG = LogFactory.getLog(MetaStoreDirectSql.class); private final PersistenceManager pm; + private final String schema; /** * We want to avoid db-specific code in this class and stick with ANSI SQL. However: * 1) mysql and postgres are differently ansi-incompatible (mysql by default doesn't support @@ -116,8 +117,9 @@ private final boolean isCompatibleDatastore; private final boolean isAggregateStatsCacheEnabled; private AggregateStatsCache aggrStatsCache; - public MetaStoreDirectSql(PersistenceManager pm, Configuration conf) { + public MetaStoreDirectSql(PersistenceManager pm, Configuration conf, String schema) { this.pm = pm; + this.schema = schema; this.dbType = determineDbType(); int batchSize = HiveConf.getIntVar(conf, ConfVars.METASTORE_DIRECT_SQL_PARTITION_BATCH_SIZE); if (batchSize == DETECT_BATCHING) { @@ -145,6 +147,10 @@ public MetaStoreDirectSql(PersistenceManager pm, Configuration conf) { aggrStatsCache = AggregateStatsCache.getInstance(conf); } } + + public MetaStoreDirectSql(PersistenceManager pm, Configuration conf) { + this(pm, conf, ""); + } private DB determineDbType() { DB dbType = DB.OTHER; @@ -227,7 +233,7 @@ private boolean runTestQuery() { } Query query = null; // Run a self-test query. If it doesn't work, we will self-disable. What a PITA... - String selfTestQuery = "select \"DB_ID\" from \"DBS\""; + String selfTestQuery = "select \"DB_ID\" from "+getFullyQualifiedName("DBS"); try { doDbSpecificInitializationsBeforeQuery(); query = pm.newQuery("javax.jdo.query.SQL", selfTestQuery); @@ -248,6 +254,17 @@ private boolean runTestQuery() { } } + public String getSchema() { + return schema; + } + + private String getFullyQualifiedName(String tblName) { + if (getSchema().isEmpty()) + return "\"" + tblName + "\""; + else + return "\"" + getSchema() + "\".\"" + tblName + "\""; + } + public boolean isCompatibleDatastore() { return isCompatibleDatastore; } @@ -290,7 +307,7 @@ public Database getDatabase(String dbName) throws MetaException{ String queryTextDbSelector= "select " + "\"DB_ID\", \"NAME\", \"DB_LOCATION_URI\", \"DESC\", " + "\"OWNER_NAME\", \"OWNER_TYPE\" " - + "FROM \"DBS\" where \"NAME\" = ? "; + + "FROM "+getFullyQualifiedName("DBS")+" where \"NAME\" = ? "; Object[] params = new Object[] { dbName }; queryDbSelector = pm.newQuery("javax.jdo.query.SQL", queryTextDbSelector); @@ -314,7 +331,7 @@ public Database getDatabase(String dbName) throws MetaException{ Long dbid = extractSqlLong(dbline[0]); String queryTextDbParams = "select \"PARAM_KEY\", \"PARAM_VALUE\" " - + " FROM \"DATABASE_PARAMS\" " + + " FROM "+getFullyQualifiedName("DATABASE_PARAMS") + " WHERE \"DB_ID\" = ? " + " AND \"PARAM_KEY\" IS NOT NULL"; params[0] = dbid; @@ -371,7 +388,8 @@ public Database getDatabase(String dbName) throws MetaException{ return new ArrayList(); } return getPartitionsViaSqlFilterInternal(dbName, tblName, null, - "\"PARTITIONS\".\"PART_NAME\" in (" + makeParams(partNames.size()) + ")", + getFullyQualifiedName("PARTITIONS") + ".\"PART_NAME\" in (" + + makeParams(partNames.size()) + ")", partNames, new ArrayList(), null); } @@ -390,7 +408,7 @@ public Database getDatabase(String dbName) throws MetaException{ // Derby and Oracle do not interpret filters ANSI-properly in some cases and need a workaround. boolean dbHasJoinCastBug = (dbType == DB.DERBY || dbType == DB.ORACLE); String sqlFilter = PartitionFilterGenerator.generateSqlFilter( - table, tree, params, joins, dbHasJoinCastBug); + table, tree, params, joins, dbHasJoinCastBug, schema); if (sqlFilter == null) { return null; // Cannot make SQL filter to push down. } @@ -420,9 +438,12 @@ private static Boolean isViewTable(Table t) { private boolean isViewTable(String dbName, String tblName) throws MetaException { Query query = null; try { - String queryText = "select \"TBL_TYPE\" from \"TBLS\"" + - " inner join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\" " + - " where \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ?"; + String queryText = "select \"TBL_TYPE\" from " + getFullyQualifiedName("TBLS") + + " inner join " + getFullyQualifiedName("DBS") + " on " + + getFullyQualifiedName("TBLS") + ".\"DB_ID\" = " + + getFullyQualifiedName("DBS") + ".\"DB_ID\" " + + " where " + getFullyQualifiedName("TBLS") + ".\"TBL_NAME\" = ? and " + + getFullyQualifiedName("DBS") + ".\"NAME\" = ?"; Object[] params = new Object[] { tblName, dbName }; query = pm.newQuery("javax.jdo.query.SQL", queryText); query.setUnique(true); @@ -469,11 +490,16 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException // just adding a \"PART_ID\" IN (...) filter that doesn't alter the results to it, probably // causing it to not sort the entire table due to not knowing how selective the filter is. String queryText = - "select \"PARTITIONS\".\"PART_ID\" from \"PARTITIONS\"" - + " inner join \"TBLS\" on \"PARTITIONS\".\"TBL_ID\" = \"TBLS\".\"TBL_ID\" " - + " and \"TBLS\".\"TBL_NAME\" = ? " - + " inner join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\" " - + " and \"DBS\".\"NAME\" = ? " + "select " + getFullyQualifiedName("PARTITIONS") + ".\"PART_ID\" " + + "from " + getFullyQualifiedName("PARTITIONS") + + " inner join " + getFullyQualifiedName("TBLS") + " on " + + getFullyQualifiedName("PARTITIONS") + ".\"TBL_ID\" = " + + getFullyQualifiedName("TBLS") + ".\"TBL_ID\" " + + " and " + getFullyQualifiedName("TBLS") + ".\"TBL_NAME\" = ? " + + " inner join " + getFullyQualifiedName("DBS") + " on " + + getFullyQualifiedName("TBLS") + ".\"DB_ID\" = " + + getFullyQualifiedName("DBS") + ".\"DB_ID\" " + + " and " + getFullyQualifiedName("DBS") + ".\"NAME\" = ? " + join(joinsForFilter, ' ') + (StringUtils.isBlank(sqlFilter) ? "" : (" where " + sqlFilter)) + orderForFilter; Object[] params = new Object[paramsForFilter.size() + 2]; @@ -528,14 +554,26 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException // Get most of the fields for the IDs provided. // Assume db and table names are the same for all partition, as provided in arguments. String queryText = - "select \"PARTITIONS\".\"PART_ID\", \"SDS\".\"SD_ID\", \"SDS\".\"CD_ID\"," - + " \"SERDES\".\"SERDE_ID\", \"PARTITIONS\".\"CREATE_TIME\"," - + " \"PARTITIONS\".\"LAST_ACCESS_TIME\", \"SDS\".\"INPUT_FORMAT\", \"SDS\".\"IS_COMPRESSED\"," - + " \"SDS\".\"IS_STOREDASSUBDIRECTORIES\", \"SDS\".\"LOCATION\", \"SDS\".\"NUM_BUCKETS\"," - + " \"SDS\".\"OUTPUT_FORMAT\", \"SERDES\".\"NAME\", \"SERDES\".\"SLIB\" " - + "from \"PARTITIONS\"" - + " left outer join \"SDS\" on \"PARTITIONS\".\"SD_ID\" = \"SDS\".\"SD_ID\" " - + " left outer join \"SERDES\" on \"SDS\".\"SERDE_ID\" = \"SERDES\".\"SERDE_ID\" " + "select " + getFullyQualifiedName("PARTITIONS") + ".\"PART_ID\", " + + getFullyQualifiedName("SDS") + ".\"SD_ID\", " + getFullyQualifiedName("SDS") + ".\"CD_ID\", " + + getFullyQualifiedName("SERDES") + ".\"SERDE_ID\", " + + getFullyQualifiedName("PARTITIONS") + ".\"CREATE_TIME\", " + + getFullyQualifiedName("PARTITIONS") + ".\"LAST_ACCESS_TIME\", "+ getFullyQualifiedName("SDS") + ".\"LOCATION\", " + + getFullyQualifiedName("SDS") + ".\"INPUT_FORMAT\", " + + getFullyQualifiedName("SDS") + ".\"IS_COMPRESSED\", " + + getFullyQualifiedName("SDS") + ".\"IS_STOREDASSUBDIRECTORIES\", " + + getFullyQualifiedName("SDS") + ".\"LOCATION\", " + + getFullyQualifiedName("SDS") + ".\"NUM_BUCKETS\", " + + getFullyQualifiedName("SDS") + ".\"OUTPUT_FORMAT\", " + + getFullyQualifiedName("SERDES") + ".\"NAME\", " + + getFullyQualifiedName("SERDES") + ".\"SLIB\" " + + "from " + getFullyQualifiedName("PARTITIONS") + + " left outer join " + getFullyQualifiedName("SDS") + " on " + + getFullyQualifiedName("PARTITIONS") + ".\"SD_ID\" = " + + getFullyQualifiedName("SDS") + ".\"SD_ID\" " + + " left outer join " + getFullyQualifiedName("SERDES") + " on " + + getFullyQualifiedName("SDS") + ".\"SERDE_ID\" = " + + getFullyQualifiedName("SERDES") + ".\"SERDE_ID\" " + "where \"PART_ID\" in (" + partIds + ") order by \"PART_NAME\" asc"; long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); @@ -636,7 +674,8 @@ private boolean isViewTable(String dbName, String tblName) throws MetaException timingTrace(doTrace, queryText, start, queryTime); // Now get all the one-to-many things. Start with partitions. - queryText = "select \"PART_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from \"PARTITION_PARAMS\"" + queryText = "select \"PART_ID\", \"PARAM_KEY\", \"PARAM_VALUE\"" + + " from " + getFullyQualifiedName("PARTITION_PARAMS") + " where \"PART_ID\" in (" + partIds + ") and \"PARAM_KEY\" is not null" + " order by \"PART_ID\" asc"; loopJoinOrderedResult(partitions, queryText, 0, new ApplyFunc() { @@ -645,7 +684,8 @@ public void apply(Partition t, Object[] fields) { t.putToParameters((String)fields[1], (String)fields[2]); }}); - queryText = "select \"PART_ID\", \"PART_KEY_VAL\" from \"PARTITION_KEY_VALS\"" + queryText = "select \"PART_ID\", \"PART_KEY_VAL\"" + + " from " + getFullyQualifiedName("PARTITION_KEY_VALS") + " where \"PART_ID\" in (" + partIds + ") and \"INTEGER_IDX\" >= 0" + " order by \"PART_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(partitions, queryText, 0, new ApplyFunc() { @@ -663,7 +703,8 @@ public void apply(Partition t, Object[] fields) { colIds = trimCommaList(colsSb); // Get all the stuff for SD. Don't do empty-list check - we expect partitions do have SDs. - queryText = "select \"SD_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from \"SD_PARAMS\"" + queryText = "select \"SD_ID\", \"PARAM_KEY\", \"PARAM_VALUE\"" + + " from " + getFullyQualifiedName("SD_PARAMS") + " where \"SD_ID\" in (" + sdIds + ") and \"PARAM_KEY\" is not null" + " order by \"SD_ID\" asc"; loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { @@ -672,7 +713,9 @@ public void apply(StorageDescriptor t, Object[] fields) { t.putToParameters((String)fields[1], (String)fields[2]); }}); - queryText = "select \"SD_ID\", \"COLUMN_NAME\", \"SORT_COLS\".\"ORDER\" from \"SORT_COLS\"" + queryText = "select \"SD_ID\", \"COLUMN_NAME\", " + + getFullyQualifiedName("SORT_COLS") + ".\"ORDER\"" + + " from " + getFullyQualifiedName("SORT_COLS") + " where \"SD_ID\" in (" + sdIds + ") and \"INTEGER_IDX\" >= 0" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { @@ -682,7 +725,8 @@ public void apply(StorageDescriptor t, Object[] fields) { t.addToSortCols(new Order((String)fields[1], extractSqlInt(fields[2]))); }}); - queryText = "select \"SD_ID\", \"BUCKET_COL_NAME\" from \"BUCKETING_COLS\"" + queryText = "select \"SD_ID\", \"BUCKET_COL_NAME\"" + + " from " + getFullyQualifiedName("BUCKETING_COLS") + " where \"SD_ID\" in (" + sdIds + ") and \"INTEGER_IDX\" >= 0" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { @@ -692,7 +736,8 @@ public void apply(StorageDescriptor t, Object[] fields) { }}); // Skewed columns stuff. - queryText = "select \"SD_ID\", \"SKEWED_COL_NAME\" from \"SKEWED_COL_NAMES\"" + queryText = "select \"SD_ID\", \"SKEWED_COL_NAME\"" + + " from " + getFullyQualifiedName("SKEWED_COL_NAMES") + " where \"SD_ID\" in (" + sdIds + ") and \"INTEGER_IDX\" >= 0" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; boolean hasSkewedColumns = @@ -707,17 +752,19 @@ public void apply(StorageDescriptor t, Object[] fields) { if (hasSkewedColumns) { // We are skipping the SKEWED_STRING_LIST table here, as it seems to be totally useless. queryText = - "select \"SKEWED_VALUES\".\"SD_ID_OID\"," - + " \"SKEWED_STRING_LIST_VALUES\".\"STRING_LIST_ID\"," - + " \"SKEWED_STRING_LIST_VALUES\".\"STRING_LIST_VALUE\" " - + "from \"SKEWED_VALUES\" " - + " left outer join \"SKEWED_STRING_LIST_VALUES\" on \"SKEWED_VALUES\"." - + "\"STRING_LIST_ID_EID\" = \"SKEWED_STRING_LIST_VALUES\".\"STRING_LIST_ID\" " - + "where \"SKEWED_VALUES\".\"SD_ID_OID\" in (" + sdIds + ") " - + " and \"SKEWED_VALUES\".\"STRING_LIST_ID_EID\" is not null " - + " and \"SKEWED_VALUES\".\"INTEGER_IDX\" >= 0 " - + "order by \"SKEWED_VALUES\".\"SD_ID_OID\" asc, \"SKEWED_VALUES\".\"INTEGER_IDX\" asc," - + " \"SKEWED_STRING_LIST_VALUES\".\"INTEGER_IDX\" asc"; + "select " + getFullyQualifiedName("SKEWED_VALUES") + ".\"SD_ID_OID\", " + + getFullyQualifiedName("SKEWED_STRING_LIST_VALUES") + ".\"STRING_LIST_ID\", " + + getFullyQualifiedName("SKEWED_STRING_LIST_VALUES") + ".\"STRING_LIST_VALUE\" " + + "from " + getFullyQualifiedName("SKEWED_VALUES") + " " + + " left outer join " + getFullyQualifiedName("SKEWED_STRING_LIST_VALUES") + + " on " + getFullyQualifiedName("SKEWED_VALUES") + ".\"STRING_LIST_ID_EID\"" + + " = " + getFullyQualifiedName("SKEWED_STRING_LIST_VALUES") + ".\"STRING_LIST_ID\" " + + "where " + getFullyQualifiedName("SKEWED_VALUES") + ".\"SD_ID_OID\" in (" + sdIds + ") " + + " and " + getFullyQualifiedName("SKEWED_VALUES") + ".\"STRING_LIST_ID_EID\" is not null " + + " and " + getFullyQualifiedName("SKEWED_VALUES") + ".\"INTEGER_IDX\" >= 0 " + + "order by " + getFullyQualifiedName("SKEWED_VALUES") + ".\"SD_ID_OID\" asc, " + + getFullyQualifiedName("SKEWED_VALUES") + ".\"INTEGER_IDX\" asc, " + + getFullyQualifiedName("SKEWED_STRING_LIST_VALUES") + ".\"INTEGER_IDX\" asc"; loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { private Long currentListId; private List currentList; @@ -743,18 +790,20 @@ public void apply(StorageDescriptor t, Object[] fields) throws MetaException { // We are skipping the SKEWED_STRING_LIST table here, as it seems to be totally useless. queryText = - "select \"SKEWED_COL_VALUE_LOC_MAP\".\"SD_ID\"," - + " \"SKEWED_STRING_LIST_VALUES\".STRING_LIST_ID," - + " \"SKEWED_COL_VALUE_LOC_MAP\".\"LOCATION\"," - + " \"SKEWED_STRING_LIST_VALUES\".\"STRING_LIST_VALUE\" " - + "from \"SKEWED_COL_VALUE_LOC_MAP\"" - + " left outer join \"SKEWED_STRING_LIST_VALUES\" on \"SKEWED_COL_VALUE_LOC_MAP\"." - + "\"STRING_LIST_ID_KID\" = \"SKEWED_STRING_LIST_VALUES\".\"STRING_LIST_ID\" " - + "where \"SKEWED_COL_VALUE_LOC_MAP\".\"SD_ID\" in (" + sdIds + ")" - + " and \"SKEWED_COL_VALUE_LOC_MAP\".\"STRING_LIST_ID_KID\" is not null " - + "order by \"SKEWED_COL_VALUE_LOC_MAP\".\"SD_ID\" asc," - + " \"SKEWED_STRING_LIST_VALUES\".\"STRING_LIST_ID\" asc," - + " \"SKEWED_STRING_LIST_VALUES\".\"INTEGER_IDX\" asc"; + "select " + getFullyQualifiedName("SKEWED_COL_VALUE_LOC_MAP") + ".\"SD_ID\", " + + getFullyQualifiedName("SKEWED_STRING_LIST_VALUES") + ".\"STRING_LIST_ID\", " + + getFullyQualifiedName("SKEWED_COL_VALUE_LOC_MAP") + ".\"LOCATION\", " + + getFullyQualifiedName("SKEWED_STRING_LIST_VALUES") + ".\"STRING_LIST_VALUE\" " + + "from " + getFullyQualifiedName("SKEWED_COL_VALUE_LOC_MAP") + + " left outer join " + getFullyQualifiedName("SKEWED_STRING_LIST_VALUES") + + " on " + getFullyQualifiedName("SKEWED_COL_VALUE_LOC_MAP") + ".\"STRING_LIST_ID_KID\"" + + " = " + getFullyQualifiedName("SKEWED_STRING_LIST_VALUES") + ".\"STRING_LIST_ID\" " + + "where " + getFullyQualifiedName("SKEWED_COL_VALUE_LOC_MAP") + ".\"SD_ID\" in (" + sdIds + ")" + + " and " + getFullyQualifiedName("SKEWED_COL_VALUE_LOC_MAP") + ".\"STRING_LIST_ID_KID\"" + + " is not null " + + "order by " + getFullyQualifiedName("SKEWED_COL_VALUE_LOC_MAP") + ".\"SD_ID\" asc, " + + getFullyQualifiedName("SKEWED_STRING_LIST_VALUES") + ".\"STRING_LIST_ID\" asc, " + + getFullyQualifiedName("SKEWED_STRING_LIST_VALUES") + ".\"INTEGER_IDX\" asc"; loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { private Long currentListId; @@ -790,7 +839,8 @@ public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!colss.isEmpty()) { // We are skipping the CDS table here, as it seems to be totally useless. queryText = "select \"CD_ID\", \"COMMENT\", \"COLUMN_NAME\", \"TYPE_NAME\"" - + " from \"COLUMNS_V2\" where \"CD_ID\" in (" + colIds + ") and \"INTEGER_IDX\" >= 0" + + " from " + getFullyQualifiedName("COLUMNS_V2") + + " where \"CD_ID\" in (" + colIds + ") and \"INTEGER_IDX\" >= 0" + " order by \"CD_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(colss, queryText, 0, new ApplyFunc>() { @Override @@ -800,7 +850,8 @@ public void apply(List t, Object[] fields) { } // Finally, get all the stuff for serdes - just the params. - queryText = "select \"SERDE_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from \"SERDE_PARAMS\"" + queryText = "select \"SERDE_ID\", \"PARAM_KEY\", \"PARAM_VALUE\"" + + " from " + getFullyQualifiedName("SERDE_PARAMS") + " where \"SERDE_ID\" in (" + serdeIds + ") and \"PARAM_KEY\" is not null" + " order by \"SERDE_ID\" asc"; loopJoinOrderedResult(serdes, queryText, 0, new ApplyFunc() { @@ -921,14 +972,24 @@ private static String trimCommaList(StringBuilder sb) { private final List params; private final List joins; private final boolean dbHasJoinCastBug; + private final String schema; private PartitionFilterGenerator( - Table table, List params, List joins, boolean dbHasJoinCastBug) { + Table table, List params, List joins, boolean dbHasJoinCastBug, + String schema) { this.table = table; this.params = params; this.joins = joins; this.dbHasJoinCastBug = dbHasJoinCastBug; this.filterBuffer = new FilterBuilder(false); + this.schema = schema; + } + + private String getFullyQualifiedName(String tblName) { + if (schema.isEmpty()) + return "\"" + tblName + "\""; + else + return "\"" + schema + "\".\"" + tblName + "\""; } /** @@ -936,16 +997,18 @@ private PartitionFilterGenerator( * @param table the table being queried * @param params the ordered parameters for the resulting expression * @param joins the joins necessary for the resulting expression + * @param schema the schema for the table * @return the string representation of the expression tree */ private static String generateSqlFilter(Table table, ExpressionTree tree, - List params, List joins, boolean dbHasJoinCastBug) throws MetaException { + List params, List joins, boolean dbHasJoinCastBug, String schema) + throws MetaException { assert table != null; if (tree.getRoot() == null) { return ""; } PartitionFilterGenerator visitor = new PartitionFilterGenerator( - table, params, joins, dbHasJoinCastBug); + table, params, joins, dbHasJoinCastBug, schema); tree.accept(visitor); if (visitor.filterBuffer.hasError()) { LOG.info("Unable to push down SQL filter: " + visitor.filterBuffer.getErrorMessage()); @@ -1064,8 +1127,10 @@ public void visit(LeafNode node) throws MetaException { } } if (joins.get(partColIndex) == null) { - joins.set(partColIndex, "inner join \"PARTITION_KEY_VALS\" \"FILTER" + partColIndex - + "\" on \"FILTER" + partColIndex + "\".\"PART_ID\" = \"PARTITIONS\".\"PART_ID\"" + joins.set(partColIndex, "inner join " + + getFullyQualifiedName("PARTITION_KEY_VALS") + " \"FILTER" + partColIndex + + "\" on \"FILTER" + partColIndex + "\".\"PART_ID\"" + + " = " + getFullyQualifiedName("PARTITIONS") + ".\"PART_ID\"" + " and \"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex); } @@ -1086,9 +1151,10 @@ public void visit(LeafNode node) throws MetaException { if (dbHasJoinCastBug) { // This is a workaround for DERBY-6358 and Oracle bug; it is pretty horrible. - tableValue = "(case when \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ? and " - + "\"FILTER" + partColIndex + "\".\"PART_ID\" = \"PARTITIONS\".\"PART_ID\" and " - + "\"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex + " then " + tableValue = "(case when " + getFullyQualifiedName("TBLS") + ".\"TBL_NAME\" = ?" + + " and " + getFullyQualifiedName("DBS") + ".\"NAME\" = ? and " + + "\"FILTER" + partColIndex + "\".\"PART_ID\" = " + getFullyQualifiedName("PARTITIONS") + ".\"PART_ID\" and " + + "\"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex + " then " + tableValue + " else null end)"; params.add(table.getTableName().toLowerCase()); params.add(table.getDbName().toLowerCase()); @@ -1112,7 +1178,8 @@ public ColumnStatistics getTableStats( doDbSpecificInitializationsBeforeQuery(); boolean doTrace = LOG.isDebugEnabled(); long start = doTrace ? System.nanoTime() : 0; - String queryText = "select " + STATS_COLLIST + " from \"TAB_COL_STATS\" " + String queryText = "select " + STATS_COLLIST + + " from " + getFullyQualifiedName("TAB_COL_STATS") + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")"; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); @@ -1204,7 +1271,7 @@ private long partsFoundForPartitions(String dbName, String tableName, assert !colNames.isEmpty() && !partNames.isEmpty(); long partsFound = 0; boolean doTrace = LOG.isDebugEnabled(); - String queryText = "select count(\"COLUMN_NAME\") from \"PART_COL_STATS\"" + String queryText = "select count(\"COLUMN_NAME\") from " + getFullyQualifiedName("PART_COL_STATS") + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")" + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" @@ -1292,7 +1359,8 @@ private long partsFoundForPartitions(String dbName, String tableName, // We need to extrapolate this partition based on the other partitions List colStats = new ArrayList(colNames.size()); queryText = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", count(\"PARTITION_NAME\") " - + " from \"PART_COL_STATS\"" + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + + " from " + getFullyQualifiedName("PART_COL_STATS") + + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")" + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\""; @@ -1357,7 +1425,7 @@ private long partsFoundForPartitions(String dbName, String tableName, // get sum for all columns to reduce the number of queries Map> sumMap = new HashMap>(); queryText = "select \"COLUMN_NAME\", sum(\"NUM_NULLS\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\"), sum(\"NUM_DISTINCTS\")" - + " from \"PART_COL_STATS\"" + + " from " + getFullyQualifiedName("PART_COL_STATS") + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + " and \"COLUMN_NAME\" in (" + makeParams(extraColumnNameTypeParts.size()) @@ -1434,13 +1502,13 @@ private long partsFoundForPartitions(String dbName, String tableName, // left/right borders if (!decimal) { queryText = "select \"" + colStatName - + "\",\"PARTITION_NAME\" from \"PART_COL_STATS\"" + + "\",\"PARTITION_NAME\" from " + getFullyQualifiedName("PART_COL_STATS") + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?" + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + " order by \"" + colStatName + "\""; } else { queryText = "select \"" + colStatName - + "\",\"PARTITION_NAME\" from \"PART_COL_STATS\"" + + "\",\"PARTITION_NAME\" from " + getFullyQualifiedName("PART_COL_STATS") + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?" + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + " order by cast(\"" + colStatName + "\" as decimal)"; @@ -1472,7 +1540,8 @@ private long partsFoundForPartitions(String dbName, String tableName, + "avg((\"LONG_HIGH_VALUE\"-\"LONG_LOW_VALUE\")/cast(\"NUM_DISTINCTS\" as decimal))," + "avg((\"DOUBLE_HIGH_VALUE\"-\"DOUBLE_LOW_VALUE\")/\"NUM_DISTINCTS\")," + "avg((cast(\"BIG_DECIMAL_HIGH_VALUE\" as decimal)-cast(\"BIG_DECIMAL_LOW_VALUE\" as decimal))/\"NUM_DISTINCTS\")" - + " from \"PART_COL_STATS\"" + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + + " from " + getFullyQualifiedName("PART_COL_STATS") + + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?" + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + " group by \"COLUMN_NAME\""; start = doTrace ? System.nanoTime() : 0; @@ -1548,7 +1617,8 @@ private ColumnStatisticsObj prepareCSObjWithAdjustedNDV(Object[] row, int i, boolean doTrace = LOG.isDebugEnabled(); doDbSpecificInitializationsBeforeQuery(); long start = doTrace ? System.nanoTime() : 0; - String queryText = "select \"PARTITION_NAME\", " + STATS_COLLIST + " from \"PART_COL_STATS\"" + String queryText = "select \"PARTITION_NAME\", " + STATS_COLLIST + + " from " + getFullyQualifiedName("PART_COL_STATS") + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ") AND \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ") order by \"PARTITION_NAME\""; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index d165fc8..94af0a9 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -332,7 +332,12 @@ private void initialize(Properties dsProps) { if (isInitialized) { expressionProxy = createExpressionProxy(hiveConf); if (HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL)) { - directSql = new MetaStoreDirectSql(pm, hiveConf); + if (prop.containsKey("javax.jdo.mapping.Schema") && + !prop.getProperty("javax.jdo.mapping.Schema").isEmpty()) { + directSql = new MetaStoreDirectSql(pm, hiveConf, + prop.getProperty("javax.jdo.mapping.Schema")); + } else + directSql = new MetaStoreDirectSql(pm, hiveConf); } } LOG.debug("RawStore: " + this + ", with PersistenceManager: " + pm +