diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 571c789edd..79f923632a 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -26,8 +26,8 @@ import java.sql.Blob; import java.sql.Clob; import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.SQLException; -import java.sql.Statement; import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; @@ -306,20 +306,20 @@ public boolean isCompatibleDatastore() { return isCompatibleDatastore; } - private void executeNoResult(final String queryText) throws SQLException { + private void executeNoResult(final String queryText, Object[] params) throws SQLException { JDOConnection jdoConn = pm.getDataStoreConnection(); - Statement statement = null; - boolean doTrace = LOG.isDebugEnabled(); - try { + try (PreparedStatement statement = ((Connection) jdoConn.getNativeConnection()).prepareStatement(queryText)) { + boolean doTrace = LOG.isDebugEnabled(); long start = doTrace ? System.nanoTime() : 0; - statement = ((Connection)jdoConn.getNativeConnection()).createStatement(); - statement.execute(queryText); - timingTrace(doTrace, queryText, start, doTrace ? System.nanoTime() : 0); - } finally { - if(statement != null){ - statement.close(); + if (params != null) { + for (int i = 0; i < params.length; i++) { + statement.setObject(i + 1, params[i]); + } } - jdoConn.close(); // We must release the connection before we call other pm methods. + statement.execute(); + timingTrace(doTrace, queryText, params, start, doTrace ? System.nanoTime() : 0); + } finally { + jdoConn.close(); } } @@ -619,7 +619,7 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw } List sqlResult = executeWithArray(query, params, queryText); long queryTime = doTrace ? System.nanoTime() : 0; - timingTrace(doTrace, queryText, start, queryTime); + timingTrace(doTrace, queryText, params, start, queryTime); if (sqlResult.isEmpty()) { return Collections.emptyList(); // no partitions, bail early. } @@ -640,7 +640,8 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw int idStringWidth = (int)Math.ceil(Math.log10(partIdList.size())) + 1; // 1 for comma int sbCapacity = partIdList.size() * idStringWidth; - String partIds = getIdListForIn(partIdList); + Object[] partIdParams = partIdList.toArray(new Object[partIdList.size()]); + String partIdParamsPlaceholder = makeParams(partIdParams.length); // Get most of the fields for the IDs provided. // Assume db and table names are the same for all partition, as provided in arguments. @@ -653,10 +654,12 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw + ".\"WRITE_ID\"" + " from " + PARTITIONS + "" + " left outer join " + SDS + " on " + PARTITIONS + ".\"SD_ID\" = " + SDS + ".\"SD_ID\" " + " left outer join " + SERDES + " on " + SDS + ".\"SERDE_ID\" = " + SERDES + ".\"SERDE_ID\" " - + "where \"PART_ID\" in (" + partIds + ") order by \"PART_NAME\" asc"; + + "where \"PART_ID\" in (" + partIdParamsPlaceholder + ") order by \"PART_NAME\" asc"; long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); - List sqlResult = executeWithArray(query, null, queryText); + List sqlResult = executeWithArray(query, + partIdParams, + queryText); long queryTime = doTrace ? System.nanoTime() : 0; Deadline.checkTimeout(); @@ -668,9 +671,10 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw // Keep order by name, consistent with JDO. ArrayList orderedResult = new ArrayList(partIdList.size()); - // Prepare StringBuilder-s for "in (...)" lists to use in one-to-many queries. - StringBuilder sdSb = new StringBuilder(sbCapacity), serdeSb = new StringBuilder(sbCapacity); - StringBuilder colsSb = new StringBuilder(7); // We expect that there's only one field schema. + List sdList = new ArrayList<>(sbCapacity); + List serdeList = new ArrayList<>(sbCapacity); + List colsList = new ArrayList<>(7); + tblName = tblName.toLowerCase(); dbName = dbName.toLowerCase(); catName = normalizeSpace(catName).toLowerCase(); @@ -731,7 +735,7 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw sd.setLocation((String)fields[9]); if (fields[10] != null) sd.setNumBuckets(extractSqlInt(fields[10])); sd.setOutputFormat((String)fields[11]); - sdSb.append(sdId).append(","); + sdList.add(sdId); part.setSd(sd); if (colId != null) { @@ -740,7 +744,7 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw if (cols == null) { cols = new ArrayList(); colss.put(colId, cols); - colsSb.append(colId).append(","); + colsList.add(colId); } sd.setCols(cols); } @@ -754,19 +758,19 @@ private boolean isViewTable(String catName, String dbName, String tblName) throw serde.setParameters(new HashMap()); serde.setName((String)fields[12]); serde.setSerializationLib((String)fields[13]); - serdeSb.append(serdeId).append(","); + serdeList.add(serdeId); sd.setSerdeInfo(serde); Deadline.checkTimeout(); } query.closeAll(); - timingTrace(doTrace, queryText, start, queryTime); + timingTrace(doTrace, queryText, partIdParams, start, queryTime); // Now get all the one-to-many things. Start with partitions. queryText = "select \"PART_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + PARTITION_PARAMS + "" - + " where \"PART_ID\" in (" + partIds + ") and \"PARAM_KEY\" is not null" + + " where \"PART_ID\" in (" + partIdParamsPlaceholder + ") and \"PARAM_KEY\" is not null" + " order by \"PART_ID\" asc"; - loopJoinOrderedResult(partitions, queryText, 0, new ApplyFunc() { + loopJoinOrderedResult(partitions, queryText, partIdParams, 0, new ApplyFunc() { @Override public void apply(Partition t, Object[] fields) { t.putToParameters((String)fields[1], extractSqlClob(fields[2])); @@ -777,29 +781,31 @@ public void apply(Partition t, Object[] fields) { } queryText = "select \"PART_ID\", \"PART_KEY_VAL\" from " + PARTITION_KEY_VALS + "" - + " where \"PART_ID\" in (" + partIds + ")" + + " where \"PART_ID\" in (" + partIdParamsPlaceholder + ")" + " order by \"PART_ID\" asc, \"INTEGER_IDX\" asc"; - loopJoinOrderedResult(partitions, queryText, 0, new ApplyFunc() { + loopJoinOrderedResult(partitions, queryText, partIdParams, 0, new ApplyFunc() { @Override public void apply(Partition t, Object[] fields) { t.addToValues((String)fields[1]); }}); - // Prepare IN (blah) lists for the following queries. Cut off the final ','s. - if (sdSb.length() == 0) { - assert serdeSb.length() == 0 && colsSb.length() == 0; + if (sdList.size() == 0) { + assert serdeList.size() == 0 && colsList.size() == 0; return orderedResult; // No SDs, probably a view. } - String sdIds = trimCommaList(sdSb); - String serdeIds = trimCommaList(serdeSb); - String colIds = trimCommaList(colsSb); + Object[] sdParams = sdList.toArray(new Object[sdList.size()]); + String sdParamsPlaceholder = makeParams(sdParams.length); + Object[] colIdsParams = colsList.toArray(new Object[colsList.size()]); + String colIdsParamsPlaceholder = makeParams(colIdsParams.length); + Object[] serdeIdsParams = serdeList.toArray(new Object[serdeList.size()]); + String serdeIdsParamsPlaceholder = makeParams(serdeIdsParams.length); // Get all the stuff for SD. Don't do empty-list check - we expect partitions do have SDs. queryText = "select \"SD_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + SD_PARAMS + "" - + " where \"SD_ID\" in (" + sdIds + ") and \"PARAM_KEY\" is not null" + + " where \"SD_ID\" in (" + sdParamsPlaceholder + ") and \"PARAM_KEY\" is not null" + " order by \"SD_ID\" asc"; - loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { + loopJoinOrderedResult(sds, queryText, sdParams, 0, new ApplyFunc() { @Override public void apply(StorageDescriptor t, Object[] fields) { t.putToParameters((String)fields[1], extractSqlClob(fields[2])); @@ -811,9 +817,9 @@ public void apply(StorageDescriptor t, Object[] fields) { queryText = "select \"SD_ID\", \"COLUMN_NAME\", " + SORT_COLS + ".\"ORDER\"" + " from " + SORT_COLS + "" - + " where \"SD_ID\" in (" + sdIds + ")" + + " where \"SD_ID\" in (" + sdParamsPlaceholder + ")" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; - loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { + loopJoinOrderedResult(sds, queryText, sdParams, 0, new ApplyFunc() { @Override public void apply(StorageDescriptor t, Object[] fields) { if (fields[2] == null) return; @@ -821,9 +827,9 @@ public void apply(StorageDescriptor t, Object[] fields) { }}); queryText = "select \"SD_ID\", \"BUCKET_COL_NAME\" from " + BUCKETING_COLS + "" - + " where \"SD_ID\" in (" + sdIds + ")" + + " where \"SD_ID\" in (" + sdParamsPlaceholder + ")" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; - loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { + loopJoinOrderedResult(sds, queryText, sdParams, 0, new ApplyFunc() { @Override public void apply(StorageDescriptor t, Object[] fields) { t.addToBucketCols((String)fields[1]); @@ -831,10 +837,10 @@ public void apply(StorageDescriptor t, Object[] fields) { // Skewed columns stuff. queryText = "select \"SD_ID\", \"SKEWED_COL_NAME\" from " + SKEWED_COL_NAMES + "" - + " where \"SD_ID\" in (" + sdIds + ")" + + " where \"SD_ID\" in (" + sdParamsPlaceholder + ")" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; boolean hasSkewedColumns = - loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { + loopJoinOrderedResult(sds, queryText, sdParams, 0, new ApplyFunc() { @Override public void apply(StorageDescriptor t, Object[] fields) { if (!t.isSetSkewedInfo()) t.setSkewedInfo(new SkewedInfo()); @@ -851,12 +857,12 @@ public void apply(StorageDescriptor t, Object[] fields) { + "from " + SKEWED_VALUES + " " + " left outer join " + SKEWED_STRING_LIST_VALUES + " on " + SKEWED_VALUES + "." + "\"STRING_LIST_ID_EID\" = " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_ID\" " - + "where " + SKEWED_VALUES + ".\"SD_ID_OID\" in (" + sdIds + ") " + + "where " + SKEWED_VALUES + ".\"SD_ID_OID\" in (" + sdParamsPlaceholder + ") " + " and " + SKEWED_VALUES + ".\"STRING_LIST_ID_EID\" is not null " + " and " + SKEWED_VALUES + ".\"INTEGER_IDX\" >= 0 " + "order by " + SKEWED_VALUES + ".\"SD_ID_OID\" asc, " + SKEWED_VALUES + ".\"INTEGER_IDX\" asc," + " " + SKEWED_STRING_LIST_VALUES + ".\"INTEGER_IDX\" asc"; - loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { + loopJoinOrderedResult(sds, queryText, sdParams, 0, new ApplyFunc() { private Long currentListId; private List currentList; @Override @@ -888,13 +894,13 @@ public void apply(StorageDescriptor t, Object[] fields) throws MetaException { + "from " + SKEWED_COL_VALUE_LOC_MAP + "" + " left outer join " + SKEWED_STRING_LIST_VALUES + " on " + SKEWED_COL_VALUE_LOC_MAP + "." + "\"STRING_LIST_ID_KID\" = " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_ID\" " - + "where " + SKEWED_COL_VALUE_LOC_MAP + ".\"SD_ID\" in (" + sdIds + ")" + + "where " + SKEWED_COL_VALUE_LOC_MAP + ".\"SD_ID\" in (" + sdParamsPlaceholder + ")" + " and " + SKEWED_COL_VALUE_LOC_MAP + ".\"STRING_LIST_ID_KID\" is not null " + "order by " + SKEWED_COL_VALUE_LOC_MAP + ".\"SD_ID\" asc," + " " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_ID\" asc," + " " + SKEWED_STRING_LIST_VALUES + ".\"INTEGER_IDX\" asc"; - loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc() { + loopJoinOrderedResult(sds, queryText, sdParams, 0, new ApplyFunc() { private Long currentListId; private List currentList; @Override @@ -928,9 +934,9 @@ public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!colss.isEmpty()) { // We are skipping the CDS table here, as it seems to be totally useless. queryText = "select \"CD_ID\", \"COMMENT\", \"COLUMN_NAME\", \"TYPE_NAME\"" - + " from " + COLUMNS_V2 + " where \"CD_ID\" in (" + colIds + ")" + + " from " + COLUMNS_V2 + " where \"CD_ID\" in (" + colIdsParamsPlaceholder + ")" + " order by \"CD_ID\" asc, \"INTEGER_IDX\" asc"; - loopJoinOrderedResult(colss, queryText, 0, new ApplyFunc>() { + loopJoinOrderedResult(colss, queryText, colIdsParams, 0, new ApplyFunc>() { @Override public void apply(List t, Object[] fields) { t.add(new FieldSchema((String)fields[2], extractSqlClob(fields[3]), (String)fields[1])); @@ -939,9 +945,12 @@ public void apply(List t, Object[] fields) { // Finally, get all the stuff for serdes - just the params. queryText = "select \"SERDE_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + SERDE_PARAMS + "" - + " where \"SERDE_ID\" in (" + serdeIds + ") and \"PARAM_KEY\" is not null" + + " where \"SERDE_ID\" in (" + serdeIdsParamsPlaceholder + ") and \"PARAM_KEY\" is " + + "not" + + " " + + "null" + " order by \"SERDE_ID\" asc"; - loopJoinOrderedResult(serdes, queryText, 0, new ApplyFunc() { + loopJoinOrderedResult(serdes, queryText, serdeIdsParams, 0, new ApplyFunc() { @Override public void apply(SerDeInfo t, Object[] fields) { t.putToParameters((String)fields[1], extractSqlClob(fields[2])); @@ -983,15 +992,28 @@ public int getNumPartitionsViaSqlFilter(SqlFilterForPushdown filter) throws Meta query.setUnique(true); int sqlResult = extractSqlInt(query.executeWithArray(params)); long queryTime = doTrace ? System.nanoTime() : 0; - timingTrace(doTrace, queryText, start, queryTime); + timingTrace(doTrace, queryText, params, start, queryTime); return sqlResult; } - private void timingTrace(boolean doTrace, String queryText, long start, long queryTime) { + private void timingTrace(boolean doTrace, String queryText, Object[] params, long start, + long queryTime) { if (!doTrace) return; - LOG.debug("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " + - (System.nanoTime() - queryTime) / 1000000.0 + "ms, the query is [" + queryText + "]"); + StringBuilder logMsg = new StringBuilder("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " + + (System.nanoTime() - queryTime) / 1000000.0 + "ms, the query is [" + queryText + "]"); + + if (params != null) { + logMsg.append(", with parameters [ "); + boolean isFirst = true; + for (Object param : params) { + logMsg.append(isFirst ? "" : ", " + param); + isFirst = false; + } + logMsg.append("]"); + } + + LOG.debug(logMsg.toString()); } static Long extractSqlLong(Object obj) throws MetaException { @@ -1087,25 +1109,6 @@ else if (value instanceof byte[]) { } } - /** - * Helper method for preparing for "SOMETHING_ID in (...)" to use in future queries. - * @param objectIds the objectId collection - * @return The concatenated list - * @throws MetaException If the list contains wrong data - */ - private static String getIdListForIn(List objectIds) throws MetaException { - return objectIds.stream() - .map(i -> i.toString()) - .collect(Collectors.joining(",")); - } - - private static String trimCommaList(StringBuilder sb) { - if (sb.length() > 0) { - sb.setLength(sb.length() - 1); - } - return sb.toString(); - } - private abstract class ApplyFunc { public abstract void apply(Target t, Object[] fields) throws MetaException; } @@ -1116,21 +1119,23 @@ private static String trimCommaList(StringBuilder sb) { * separately for every object, which is suboptimal. * @param tree The object tree, by ID. * @param queryText The query text. + * @param parameters The query parameters. * @param keyIndex Index of the Long column corresponding to the map ID in query result rows. * @param func The function that is called on each (object,row) pair with the same id. * @return the count of results returned from the query. */ private int loopJoinOrderedResult(TreeMap tree, - String queryText, int keyIndex, ApplyFunc func) throws MetaException { + String queryText, Object[] parameters, int keyIndex, ApplyFunc func) throws MetaException { boolean doTrace = LOG.isDebugEnabled(); long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); - Object result = query.execute(); + Object result = executeWithArray(query, parameters, queryText); long queryTime = doTrace ? System.nanoTime() : 0; if (result == null) { query.closeAll(); return 0; } + List list = ensureList(result); Iterator iter = list.iterator(); Object[] fields = null; @@ -1151,7 +1156,7 @@ private static String trimCommaList(StringBuilder sb) { } int rv = list.size(); query.closeAll(); - timingTrace(doTrace, queryText, start, queryTime); + timingTrace(doTrace, queryText, parameters, start, queryTime); return rv; } @@ -1409,7 +1414,7 @@ public ColumnStatistics getTableStats(final String catName, final String dbName, long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); Object qResult = executeWithArray(query, params, queryText); - timingTrace(doTrace, queryText0 + "...)", start, (doTrace ? System.nanoTime() : 0)); + timingTrace(doTrace, queryText0 + "...)", params, start, (doTrace ? System.nanoTime() : 0)); if (qResult == null) { query.closeAll(); return null; @@ -1516,10 +1521,11 @@ private long partsFoundForPartitions( long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); try { - Object qResult = executeWithArray(query, prepareParams( - catName, dbName, tableName, inputPartNames, inputColName), queryText); + Object[] params = prepareParams( + catName, dbName, tableName, inputPartNames, inputColName); + Object qResult = executeWithArray(query, params, queryText); long end = doTrace ? System.nanoTime() : 0; - timingTrace(doTrace, queryText, start, end); + timingTrace(doTrace, queryText, params, start, end); ForwardQueryResult fqr = (ForwardQueryResult) qResult; Iterator iter = fqr.iterator(); while (iter.hasNext()) { @@ -1574,13 +1580,14 @@ private long partsFoundForPartitions( List colStatsForDB = new ArrayList(); try { query = pm.newQuery("javax.jdo.query.SQL", queryText); - qResult = executeWithArray(query, new Object[] { dbName, catName }, queryText); + Object[] params = { dbName, catName }; + qResult = executeWithArray(query, params, queryText); if (qResult == null) { query.closeAll(); return colStatsForDB; } end = doTrace ? System.nanoTime() : 0; - timingTrace(doTrace, queryText, start, end); + timingTrace(doTrace, queryText, params, start, end); List list = ensureList(qResult); for (Object[] row : list) { String tblName = (String) row[0]; @@ -1649,6 +1656,7 @@ private long partsFoundForPartitions( + "sum(\"NUM_DISTINCTS\")" + " from " + PART_COL_STATS + "" + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "; String queryText = null; + Object[] params = null; long start = 0; long end = 0; Query query = null; @@ -1663,14 +1671,14 @@ private long partsFoundForPartitions( + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\""; start = doTrace ? System.nanoTime() : 0; query = pm.newQuery("javax.jdo.query.SQL", queryText); - qResult = executeWithArray(query, prepareParams(catName, dbName, tableName, partNames, colNames), - queryText); + params = prepareParams(catName, dbName, tableName, partNames, colNames); + qResult = executeWithArray(query, params, queryText); if (qResult == null) { query.closeAll(); return Collections.emptyList(); } end = doTrace ? System.nanoTime() : 0; - timingTrace(doTrace, queryText, start, end); + timingTrace(doTrace, queryText, params, start, end); List list = ensureList(qResult); List colStats = new ArrayList(list.size()); for (Object[] row : list) { @@ -1692,10 +1700,10 @@ private long partsFoundForPartitions( + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\""; start = doTrace ? System.nanoTime() : 0; query = pm.newQuery("javax.jdo.query.SQL", queryText); - qResult = executeWithArray(query, prepareParams(catName, dbName, tableName, partNames, colNames), - queryText); + params = prepareParams(catName, dbName, tableName, partNames, colNames); + qResult = executeWithArray(query, params, queryText); end = doTrace ? System.nanoTime() : 0; - timingTrace(doTrace, queryText, start, end); + timingTrace(doTrace, queryText, params, start, end); if (qResult == null) { query.closeAll(); return Collections.emptyList(); @@ -1726,8 +1734,8 @@ private long partsFoundForPartitions( + makeParams(partNames.size()) + ")" + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\""; start = doTrace ? System.nanoTime() : 0; query = pm.newQuery("javax.jdo.query.SQL", queryText); - qResult = executeWithArray(query, - prepareParams(catName, dbName, tableName, partNames, noExtraColumnNames), queryText); + params = prepareParams(catName, dbName, tableName, partNames, noExtraColumnNames); + qResult = executeWithArray(query, params, queryText); if (qResult == null) { query.closeAll(); return Collections.emptyList(); @@ -1738,7 +1746,7 @@ private long partsFoundForPartitions( Deadline.checkTimeout(); } end = doTrace ? System.nanoTime() : 0; - timingTrace(doTrace, queryText, start, end); + timingTrace(doTrace, queryText, params, start, end); query.closeAll(); } // Extrapolation is needed for extraColumnNames. @@ -1759,8 +1767,8 @@ private long partsFoundForPartitions( query = pm.newQuery("javax.jdo.query.SQL", queryText); List extraColumnNames = new ArrayList(); extraColumnNames.addAll(extraColumnNameTypeParts.keySet()); - qResult = executeWithArray(query, - prepareParams(catName, dbName, tableName, partNames, extraColumnNames), queryText); + params = prepareParams(catName, dbName, tableName, partNames, extraColumnNames); + qResult = executeWithArray(query, params, queryText); if (qResult == null) { query.closeAll(); return Collections.emptyList(); @@ -1778,7 +1786,7 @@ private long partsFoundForPartitions( Deadline.checkTimeout(); } end = doTrace ? System.nanoTime() : 0; - timingTrace(doTrace, queryText, start, end); + timingTrace(doTrace, queryText, params, start, end); query.closeAll(); for (Map.Entry entry : extraColumnNameTypeParts.entrySet()) { Object[] row = new Object[IExtrapolatePartStatus.colStatNames.length + 2]; @@ -1836,8 +1844,8 @@ private long partsFoundForPartitions( } start = doTrace ? System.nanoTime() : 0; query = pm.newQuery("javax.jdo.query.SQL", queryText); - qResult = executeWithArray(query, - prepareParams(catName, dbName, tableName, partNames, Arrays.asList(colName)), queryText); + params = prepareParams(catName, dbName, tableName, partNames, Arrays.asList(colName)); + qResult = executeWithArray(query, params, queryText); if (qResult == null) { query.closeAll(); return Collections.emptyList(); @@ -1846,7 +1854,7 @@ private long partsFoundForPartitions( Object[] min = (Object[]) (fqr.get(0)); Object[] max = (Object[]) (fqr.get(fqr.size() - 1)); end = doTrace ? System.nanoTime() : 0; - timingTrace(doTrace, queryText, start, end); + timingTrace(doTrace, queryText, params, start, end); query.closeAll(); if (min[0] == null || max[0] == null) { row[2 + colStatIndex] = null; @@ -1865,8 +1873,8 @@ private long partsFoundForPartitions( + makeParams(partNames.size()) + ")" + " group by \"COLUMN_NAME\""; start = doTrace ? System.nanoTime() : 0; query = pm.newQuery("javax.jdo.query.SQL", queryText); - qResult = executeWithArray(query, - prepareParams(catName, dbName, tableName, partNames, Arrays.asList(colName)), queryText); + params = prepareParams(catName, dbName, tableName, partNames, Arrays.asList(colName)); + qResult = executeWithArray(query, params, queryText); if (qResult == null) { query.closeAll(); return Collections.emptyList(); @@ -1877,7 +1885,7 @@ private long partsFoundForPartitions( // "AVG_DECIMAL" row[2 + colStatIndex] = avg[colStatIndex - 12]; end = doTrace ? System.nanoTime() : 0; - timingTrace(doTrace, queryText, start, end); + timingTrace(doTrace, queryText, params, start, end); query.closeAll(); } } @@ -1950,9 +1958,9 @@ private ColumnStatisticsObj prepareCSObjWithAdjustedNDV(Object[] row, int i, makeParams(inputColNames.size()), makeParams(inputPartNames.size())); long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); - Object qResult = executeWithArray(query, prepareParams( - catName, dbName, tableName, inputPartNames, inputColNames), queryText); - timingTrace(doTrace, queryText0, start, (doTrace ? System.nanoTime() : 0)); + Object[] params = prepareParams(catName, dbName, tableName, inputPartNames, inputColNames); + Object qResult = executeWithArray(query, params, queryText); + timingTrace(doTrace, queryText0, params, start, (doTrace ? System.nanoTime() : 0)); if (qResult == null) { query.closeAll(); return Collections.emptyList(); @@ -2064,7 +2072,7 @@ public void prepareTxn() throws MetaException { if (dbType != DatabaseProduct.MYSQL) return; try { assert pm.currentTransaction().isActive(); // must be inside tx together with queries - executeNoResult("SET @@session.sql_mode=ANSI_QUOTES"); + executeNoResult("SET @@session.sql_mode=ANSI_QUOTES", null); } catch (SQLException sqlEx) { throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage()); } @@ -2502,17 +2510,19 @@ private void dropPartitionsByPartitionIds(List partitionIdList) throws M if (partitionIdList.isEmpty()) { return; } - String partitionIds = getIdListForIn(partitionIdList); + + Object[] partitionIdsParams = partitionIdList.toArray(new Object[partitionIdList.size()]); + String partitionIdsParamsPlaceholder = makeParams(partitionIdsParams.length); // Get the corresponding SD_ID-s, CD_ID-s, SERDE_ID-s queryText = "SELECT " + SDS + ".\"SD_ID\", " + SDS + ".\"CD_ID\", " + SDS + ".\"SERDE_ID\" " + "from " + SDS + " " + "INNER JOIN " + PARTITIONS + " ON " + PARTITIONS + ".\"SD_ID\" = " + SDS + ".\"SD_ID\" " - + "WHERE " + PARTITIONS + ".\"PART_ID\" in (" + partitionIds + ")"; + + "WHERE " + PARTITIONS + ".\"PART_ID\" in (" + partitionIdsParamsPlaceholder + ")"; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); - List sqlResult = ensureList(executeWithArray(query, null, queryText)); + List sqlResult = ensureList(executeWithArray(query, partitionIdsParams, queryText)); List sdIdList = new ArrayList<>(partitionIdList.size()); List columnDescriptorIdList = new ArrayList<>(1); @@ -2532,35 +2542,41 @@ private void dropPartitionsByPartitionIds(List partitionIdList) throws M try { // Drop privileges - queryText = "delete from " + PART_PRIVS + " where \"PART_ID\" in (" + partitionIds + ")"; - executeNoResult(queryText); + queryText = + "delete from " + PART_PRIVS + " where \"PART_ID\" in (" + partitionIdsParamsPlaceholder + ")"; + executeNoResult(queryText, partitionIdsParams); Deadline.checkTimeout(); // Drop column level privileges - queryText = "delete from " + PART_COL_PRIVS + " where \"PART_ID\" in (" + partitionIds + ")"; - executeNoResult(queryText); + queryText = + "delete from " + PART_COL_PRIVS + " where \"PART_ID\" in (" + partitionIdsParamsPlaceholder + + ")"; + executeNoResult(queryText, partitionIdsParams); Deadline.checkTimeout(); // Drop partition statistics - queryText = "delete from " + PART_COL_STATS + " where \"PART_ID\" in (" + partitionIds + ")"; - executeNoResult(queryText); + queryText = + "delete from " + PART_COL_STATS + " where \"PART_ID\" in (" + partitionIdsParamsPlaceholder + + ")"; + executeNoResult(queryText, partitionIdsParams); Deadline.checkTimeout(); // Drop the partition params queryText = "delete from " + PARTITION_PARAMS + " where \"PART_ID\" in (" - + partitionIds + ")"; - executeNoResult(queryText); + + partitionIdsParamsPlaceholder + ")"; + executeNoResult(queryText, partitionIdsParams); Deadline.checkTimeout(); // Drop the partition key vals queryText = "delete from " + PARTITION_KEY_VALS + " where \"PART_ID\" in (" - + partitionIds + ")"; - executeNoResult(queryText); + + partitionIdsParamsPlaceholder + ")"; + executeNoResult(queryText, partitionIdsParams); Deadline.checkTimeout(); // Drop the partitions - queryText = "delete from " + PARTITIONS + " where \"PART_ID\" in (" + partitionIds + ")"; - executeNoResult(queryText); + queryText = + "delete from " + PARTITIONS + " where \"PART_ID\" in (" + partitionIdsParamsPlaceholder + ")"; + executeNoResult(queryText, partitionIdsParams); Deadline.checkTimeout(); } catch (SQLException sqlException) { LOG.warn("SQL error executing query while dropping partition", sqlException); @@ -2586,16 +2602,18 @@ private void dropStorageDescriptors(List storageDescriptorIdList) throws return; } String queryText; - String sdIds = getIdListForIn(storageDescriptorIdList); + Object[] sdIdsParams = + storageDescriptorIdList.toArray(new Object[storageDescriptorIdList.size()]); + String sdIdsParamsPlaceholder = makeParams(sdIdsParams.length); // Get the corresponding SKEWED_STRING_LIST_ID data queryText = "select " + SKEWED_VALUES + ".\"STRING_LIST_ID_EID\" " + "from " + SKEWED_VALUES + " " - + "WHERE " + SKEWED_VALUES + ".\"SD_ID_OID\" in (" + sdIds + ")"; + + "WHERE " + SKEWED_VALUES + ".\"SD_ID_OID\" in (" + sdIdsParamsPlaceholder + ")"; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); - List sqlResult = ensureList(executeWithArray(query, null, queryText)); + List sqlResult = ensureList(executeWithArray(query, sdIdsParams, queryText)); List skewedStringListIdList = new ArrayList<>(0); @@ -2606,58 +2624,65 @@ private void dropStorageDescriptors(List storageDescriptorIdList) throws } query.closeAll(); - String skewedStringListIds = getIdListForIn(skewedStringListIdList); + Object[] skewedStringParams = + skewedStringListIdList.toArray(new Object[skewedStringListIdList.size()]); + String skewedStringParamsPlaceholder = makeParams(skewedStringParams.length); try { // Drop the SD params - queryText = "delete from " + SD_PARAMS + " where \"SD_ID\" in (" + sdIds + ")"; - executeNoResult(queryText); + queryText = + "delete from " + SD_PARAMS + " where \"SD_ID\" in (" + sdIdsParamsPlaceholder + + ")"; + executeNoResult(queryText, sdIdsParams); Deadline.checkTimeout(); // Drop the sort cols - queryText = "delete from " + SORT_COLS + " where \"SD_ID\" in (" + sdIds + ")"; - executeNoResult(queryText); + queryText = + "delete from " + SORT_COLS + " where \"SD_ID\" in (" + sdIdsParamsPlaceholder + + ")"; + executeNoResult(queryText, sdIdsParams); Deadline.checkTimeout(); // Drop the bucketing cols - queryText = "delete from " + BUCKETING_COLS + " where \"SD_ID\" in (" + sdIds + ")"; - executeNoResult(queryText); + queryText = + "delete from " + BUCKETING_COLS + " where \"SD_ID\" in (" + sdIdsParamsPlaceholder + ")"; + executeNoResult(queryText, sdIdsParams); Deadline.checkTimeout(); // Drop the skewed string lists if (skewedStringListIdList.size() > 0) { // Drop the skewed string value loc map queryText = "delete from " + SKEWED_COL_VALUE_LOC_MAP + " where \"SD_ID\" in (" - + sdIds + ")"; - executeNoResult(queryText); + + sdIdsParamsPlaceholder + ")"; + executeNoResult(queryText, sdIdsParams); Deadline.checkTimeout(); // Drop the skewed values - queryText = "delete from " + SKEWED_VALUES + " where \"SD_ID_OID\" in (" + sdIds + ")"; - executeNoResult(queryText); + queryText = "delete from " + SKEWED_VALUES + " where \"SD_ID_OID\" in (" + sdIdsParamsPlaceholder + ")"; + executeNoResult(queryText, sdIdsParams); Deadline.checkTimeout(); // Drop the skewed string list values queryText = "delete from " + SKEWED_STRING_LIST_VALUES + " where \"STRING_LIST_ID\" in (" - + skewedStringListIds + ")"; - executeNoResult(queryText); + + skewedStringParamsPlaceholder + ")"; + executeNoResult(queryText, skewedStringParams); Deadline.checkTimeout(); // Drop the skewed string list queryText = "delete from " + SKEWED_STRING_LIST + " where \"STRING_LIST_ID\" in (" - + skewedStringListIds + ")"; - executeNoResult(queryText); + + skewedStringParamsPlaceholder + ")"; + executeNoResult(queryText, skewedStringParams); Deadline.checkTimeout(); } // Drop the skewed cols - queryText = "delete from " + SKEWED_COL_NAMES + " where \"SD_ID\" in (" + sdIds + ")"; - executeNoResult(queryText); + queryText = "delete from " + SKEWED_COL_NAMES + " where \"SD_ID\" in (" + sdIdsParamsPlaceholder + ")"; + executeNoResult(queryText, sdIdsParams); Deadline.checkTimeout(); // Drop the sds - queryText = "delete from " + SDS + " where \"SD_ID\" in (" + sdIds + ")"; - executeNoResult(queryText); + queryText = "delete from " + SDS + " where \"SD_ID\" in (" + sdIdsParamsPlaceholder + ")"; + executeNoResult(queryText, sdIdsParams); } catch (SQLException sqlException) { LOG.warn("SQL error executing query while dropping storage descriptor.", sqlException); throw new MetaException("Encountered error while dropping storage descriptor."); @@ -2675,17 +2700,18 @@ private void dropSerdes(List serdeIdList) throws MetaException { if (serdeIdList.isEmpty()) { return; } - String serdeIds = getIdListForIn(serdeIdList); + Object[] serdeIdsParams = serdeIdList.toArray(new Object[serdeIdList.size()]); + String serdeIdsParamsPlaceholder = makeParams(serdeIdsParams.length); try { // Drop the serde params - queryText = "delete from " + SERDE_PARAMS + " where \"SERDE_ID\" in (" + serdeIds + ")"; - executeNoResult(queryText); + queryText = "delete from " + SERDE_PARAMS + " where \"SERDE_ID\" in (" + serdeIdsParamsPlaceholder + ")"; + executeNoResult(queryText, serdeIdsParams); Deadline.checkTimeout(); // Drop the serdes - queryText = "delete from " + SERDES + " where \"SERDE_ID\" in (" + serdeIds + ")"; - executeNoResult(queryText); + queryText = "delete from " + SERDES + " where \"SERDE_ID\" in (" + serdeIdsParamsPlaceholder + ")"; + executeNoResult(queryText, serdeIdsParams); } catch (SQLException sqlException) { LOG.warn("SQL error executing query while dropping serde.", sqlException); throw new MetaException("Encountered error while dropping serde."); @@ -2705,16 +2731,18 @@ private void dropDanglingColumnDescriptors(List columnDescriptorIdList) return; } String queryText; - String colIds = getIdListForIn(columnDescriptorIdList); + Object[] coldIdsParams = + columnDescriptorIdList.toArray(new Object[columnDescriptorIdList.size()]); + String colIdsParamsPlaceholder = makeParams(coldIdsParams.length); // Drop column descriptor, if no relation left queryText = "SELECT " + SDS + ".\"CD_ID\", count(1) " + "from " + SDS + " " - + "WHERE " + SDS + ".\"CD_ID\" in (" + colIds + ") " + + "WHERE " + SDS + ".\"CD_ID\" in (" + colIdsParamsPlaceholder + ") " + "GROUP BY " + SDS + ".\"CD_ID\""; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); - List sqlResult = ensureList(executeWithArray(query, null, queryText)); + List sqlResult = ensureList(executeWithArray(query, coldIdsParams, queryText)); List danglingColumnDescriptorIdList = new ArrayList<>(columnDescriptorIdList.size()); if (!sqlResult.isEmpty()) { @@ -2728,16 +2756,19 @@ private void dropDanglingColumnDescriptors(List columnDescriptorIdList) if (!danglingColumnDescriptorIdList.isEmpty()) { try { - String danglingCDIds = getIdListForIn(danglingColumnDescriptorIdList); + Object[] danglingCDIdsParams = + danglingColumnDescriptorIdList.toArray(new Object[danglingColumnDescriptorIdList.size()]); + String danglingCDIdsParamsPlaceholder = makeParams(danglingCDIdsParams.length); // Drop the columns_v2 - queryText = "delete from " + COLUMNS_V2 + " where \"CD_ID\" in (" + danglingCDIds + ")"; - executeNoResult(queryText); + queryText = "delete from " + COLUMNS_V2 + " where \"CD_ID\" in (" + danglingCDIdsParamsPlaceholder + ")"; + executeNoResult(queryText, danglingCDIdsParams); Deadline.checkTimeout(); // Drop the cols - queryText = "delete from " + CDS + " where \"CD_ID\" in (" + danglingCDIds + ")"; - executeNoResult(queryText); + queryText = + "delete from " + CDS + " where \"CD_ID\" in (" + danglingCDIdsParamsPlaceholder + ")"; + executeNoResult(queryText, danglingCDIdsParams); } catch (SQLException sqlException) { LOG.warn("SQL error executing query while dropping dangling col descriptions", sqlException); throw new MetaException("Encountered error while dropping col descriptions");