diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java index 4900a41..31fc407 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java @@ -182,13 +182,17 @@ public void initialize(Configuration conf, Properties tbl) String [] columnSpecs = columnsMappingSpec.split(","); ColumnMapping columnMapping = null; + String mappingSpec, colInfo; + String [] mapInfo; + String [] parts; + int idxFirst, idxLast; for (int i = 0; i < columnSpecs.length; i++) { - String mappingSpec = columnSpecs[i].trim(); - String [] mapInfo = mappingSpec.split("#"); - String colInfo = mapInfo[0]; + mappingSpec = columnSpecs[i].trim(); + mapInfo = mappingSpec.split("#"); + colInfo = mapInfo[0]; - int idxFirst = colInfo.indexOf(":"); - int idxLast = colInfo.lastIndexOf(":"); + idxFirst = colInfo.indexOf(":"); + idxLast = colInfo.lastIndexOf(":"); if (idxFirst < 0 || !(idxFirst == idxLast)) { throw new SerDeException("Error: the HBase columns mapping contains a badly formed " + @@ -205,7 +209,7 @@ public void initialize(Configuration conf, Properties tbl) columnMapping.qualifierNameBytes = null; columnMapping.hbaseRowKey = true; } else { - String [] parts = colInfo.split(":"); + parts = colInfo.split(":"); assert(parts.length > 0 && parts.length <= 2); columnMapping.familyName = parts[0]; columnMapping.familyNameBytes = Bytes.toBytes(parts[0]); diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java index abde695..2c8137d 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsAggregator.java @@ -113,8 +113,9 @@ public boolean cleanUp(String rowID) { scan.setFilter(filter); ResultScanner scanner = htable.getScanner(scan); ArrayList toDelete = new ArrayList(); + Delete delete; for (Result result : scanner) { - Delete delete = new Delete(result.getRow()); + delete = new Delete(result.getRow()); toDelete.add(delete); } htable.delete(toDelete); diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java index 766056b..3960d8a 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java @@ -303,12 +303,14 @@ private TableSplit convertFilter( // Convert the search condition into a restriction on the HBase scan byte [] startRow = HConstants.EMPTY_START_ROW, stopRow = HConstants.EMPTY_END_ROW; + ExprNodeConstantEvaluator eval; + PrimitiveObjectInspector objInspector; + Object writable; + String comparisonOp; + byte[] constantVal; for (IndexSearchCondition sc : searchConditions){ - - ExprNodeConstantEvaluator eval = new ExprNodeConstantEvaluator(sc.getConstantDesc()); - PrimitiveObjectInspector objInspector; - Object writable; - + eval = new ExprNodeConstantEvaluator(sc.getConstantDesc()); + try{ objInspector = (PrimitiveObjectInspector)eval.initialize(null); writable = eval.evaluate(null); @@ -319,8 +321,8 @@ private TableSplit convertFilter( throw new IOException(e); } - byte [] constantVal = getConstantVal(writable, objInspector, isKeyBinary); - String comparisonOp = sc.getComparisonOp(); + constantVal = getConstantVal(writable, objInspector, isKeyBinary); + comparisonOp = sc.getComparisonOp(); if("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual".equals(comparisonOp)){ startRow = constantVal; diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java index b254b0d..610534a 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java @@ -76,8 +76,9 @@ private void parse() { ((StructObjectInspector)getInspector()).getAllStructFieldRefs(); LazyObject [] fields = new LazyObject[fieldRefs.size()]; + ColumnMapping colMap; for (int i = 0; i < fields.length; i++) { - ColumnMapping colMap = columnsMapping.get(i); + colMap = columnsMapping.get(i); if (colMap.qualifierName == null && !colMap.hbaseRowKey) { // a column family diff --git a/hcatalog/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/HCatTypeCheckHive.java b/hcatalog/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/HCatTypeCheckHive.java index 61b541e..4c599f4 100644 --- a/hcatalog/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/HCatTypeCheckHive.java +++ b/hcatalog/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/HCatTypeCheckHive.java @@ -72,10 +72,14 @@ public Object evaluate(DeferredObject[] args) throws HiveException { for (int i = 0; i < types.length; i++) { types[i] = types[i].toLowerCase(); } + + ObjectInspector oi; + List categories; + Object o; for (int i = 1; i < args.length; i++) { - ObjectInspector oi = argOIs[i]; - List categories = new ArrayList(); - Object o = getJavaObject(args[i].get(), oi, categories); + oi = argOIs[i]; + categories = new ArrayList(); + o = getJavaObject(args[i].get(), oi, categories); try { if (o != null) { Util.check(types[i - 1], o); diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java index 31b7741..084d96e 100644 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java +++ b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java @@ -282,8 +282,9 @@ public void preCreateTable(Table tbl) throws MetaException { uniqueColumnFamilies.addAll(hbaseColumnFamilies); uniqueColumnFamilies.remove(hbaseColumnFamilies.get(iKey)); + HColumnDescriptor familyDesc; for (String columnFamily : uniqueColumnFamilies) { - HColumnDescriptor familyDesc = new HColumnDescriptor(Bytes + familyDesc = new HColumnDescriptor(Bytes .toBytes(columnFamily)); familyDesc.setMaxVersions(Integer.MAX_VALUE); tableDesc.addFamily(familyDesc); diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java index f4f9ed1..d5bbeb9 100644 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java +++ b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java @@ -112,10 +112,11 @@ public HBaseInputFormat() { private InputSplit[] convertSplits(List splits) { InputSplit[] converted = new InputSplit[splits.size()]; + TableSplit newTableSplit; for (int i = 0; i < splits.size(); i++) { org.apache.hadoop.hbase.mapreduce.TableSplit tableSplit = (org.apache.hadoop.hbase.mapreduce.TableSplit) splits.get(i); - TableSplit newTableSplit = new TableSplit(tableSplit.getTableName(), + newTableSplit = new TableSplit(tableSplit.getTableName(), tableSplit.getStartRow(), tableSplit.getEndRow(), tableSplit.getRegionLocation()); converted[i] = newTableSplit; diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HbaseSnapshotRecordReader.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HbaseSnapshotRecordReader.java index 6f18846..66d1c3d 100644 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HbaseSnapshotRecordReader.java +++ b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HbaseSnapshotRecordReader.java @@ -198,24 +198,28 @@ private Result prepareResult(List keyvalues) { List finalKeyVals = new ArrayList(); Map> qualValMap = new HashMap>(); + byte[] cf; + byte[] qualifier; + String key, family; + List kvs; + long desiredTS; for (KeyValue kv : keyvalues) { - byte[] cf = kv.getFamily(); - byte[] qualifier = kv.getQualifier(); - String key = Bytes.toString(cf) + ":" + Bytes.toString(qualifier); - List kvs; + cf = kv.getFamily(); + qualifier = kv.getQualifier(); + key = Bytes.toString(cf) + ":" + Bytes.toString(qualifier); if (qualValMap.containsKey(key)) { kvs = qualValMap.get(key); } else { kvs = new ArrayList(); } - String family = Bytes.toString(kv.getFamily()); + family = Bytes.toString(kv.getFamily()); //Ignore aborted transactions if (allAbortedTransactions.contains(kv.getTimestamp())) { continue; } - long desiredTS = snapshot.getRevision(family); + desiredTS = snapshot.getRevision(family); if (kv.getTimestamp() <= desiredTS) { kvs.add(kv); } @@ -223,8 +227,8 @@ private Result prepareResult(List keyvalues) { } Set keys = qualValMap.keySet(); - for (String cf : keys) { - List kvs = qualValMap.get(cf); + for (String cfk : keys) { + kvs = qualValMap.get(cfk); if (maxRevisions <= kvs.size()) { for (int i = 0; i < maxRevisions; i++) { finalKeyVals.add(kvs.get(i)); diff --git a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java index e52584d..a321f9b 100644 --- a/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java +++ b/hcatalog/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKUtil.java @@ -74,12 +74,14 @@ void setUpZnodesForTable(String table, List families) String transactionDataTablePath = PathUtil.getTxnDataPath(baseDir, table); ensurePathExists(transactionDataTablePath, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + String runningDataPath; + String abortDataPath; for (String cf : families) { - String runningDataPath = PathUtil.getRunningTxnInfoPath( + runningDataPath = PathUtil.getRunningTxnInfoPath( this.baseDir, table, cf); ensurePathExists(runningDataPath, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - String abortDataPath = PathUtil.getAbortInformationPath( + abortDataPath = PathUtil.getAbortInformationPath( this.baseDir, table, cf); ensurePathExists(abortDataPath, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); @@ -101,11 +103,12 @@ void ensurePathExists(String path, byte[] data, List acl, CreateMode flags) throws IOException { String[] dirs = path.split("/"); String parentPath = ""; + Stat stat; for (String subDir : dirs) { if (subDir.equals("") == false) { parentPath = parentPath + "/" + subDir; try { - Stat stat = getSession().exists(parentPath, false); + stat = getSession().exists(parentPath, false); if (stat == null) { getSession().create(parentPath, data, acl, flags); } @@ -234,9 +237,9 @@ long currentID(String tableName) throws IOException { StoreFamilyRevisionList txnList = new StoreFamilyRevisionList(); deserialize(txnList, data); Iterator itr = txnList.getRevisionListIterator(); - + StoreFamilyRevision wtxn; while (itr.hasNext()) { - StoreFamilyRevision wtxn = itr.next(); + wtxn = itr.next(); wtxnList.add(new FamilyRevision(wtxn.getRevision(), wtxn .getTimestamp())); } @@ -367,8 +370,9 @@ void updateData(String path, FamilyRevision updateTx, UpdateMode mode) // For serialization purposes. List newTxnList = new ArrayList(); + StoreFamilyRevision newTxn; for (FamilyRevision wtxn : newData) { - StoreFamilyRevision newTxn = new StoreFamilyRevision(wtxn.getRevision(), + newTxn = new StoreFamilyRevision(wtxn.getRevision(), wtxn.getExpireTimestamp()); newTxnList.add(newTxn); } @@ -410,8 +414,9 @@ void refreshTransactions(String path) throws IOException { if (newData.equals(currentData) == false) { List newTxnList = new ArrayList(); + StoreFamilyRevision newTxn; for (FamilyRevision wtxn : newData) { - StoreFamilyRevision newTxn = new StoreFamilyRevision(wtxn.getRevision(), + newTxn = new StoreFamilyRevision(wtxn.getRevision(), wtxn.getExpireTimestamp()); newTxnList.add(newTxn); } diff --git a/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java b/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java index a10c3a8..74056d1 100644 --- a/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java +++ b/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java @@ -342,8 +342,9 @@ public void runQuery() { queryRet.add(Integer.valueOf(qp.run(cmd).getResponseCode())); ArrayList res = new ArrayList(); try { + ArrayList resCopy; while (qp.getResults(res)) { - ArrayList resCopy = new ArrayList(); + resCopy = new ArrayList(); resCopy.addAll(res); resultBucket.add(resCopy); if (resultBucket.size() > resultBucketMaxSize) { diff --git a/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveDatabaseMetaData.java b/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveDatabaseMetaData.java index bda3e0d..65c4e5d 100644 --- a/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveDatabaseMetaData.java +++ b/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveDatabaseMetaData.java @@ -216,10 +216,12 @@ public ResultSet getColumns(String catalog, final String schemaPattern String regcolumnNamePattern = convertPattern(columnNamePattern); List tables = client.get_tables(catalog, "*"); + List fields; + int ordinalPos; for (String table: tables) { if (table.matches(regtableNamePattern)) { - List fields = client.get_schema(catalog, table); - int ordinalPos = 1; + fields = client.get_schema(catalog, table); + ordinalPos = 1; for (FieldSchema field: fields) { if (field.getName().matches(regcolumnNamePattern)) { columns.add(new JdbcColumn(field.getName(), table, catalog @@ -640,14 +642,16 @@ public ResultSet getTables(String catalog, String schemaPattern, String regtableNamePattern = convertPattern(tableNamePattern); try { tablesstr = client.get_tables(resultCatalog, "*"); + Table tbl; + String tableType; for (String tablestr: tablesstr) { if (tablestr.matches(regtableNamePattern)) { - Table tbl = client.get_table(resultCatalog, tablestr); + tbl = client.get_table(resultCatalog, tablestr); if (types == null) { resultTables.add(new JdbcTable(resultCatalog, tbl.getTableName(), tbl.getTableType() , tbl.getParameters().get("comment"))); } else { - String tableType = toJdbcTableType(tbl.getTableType()); + tableType = toJdbcTableType(tbl.getTableType()); for(String type : types) { if (type.equalsIgnoreCase(tableType)) { resultTables.add(new JdbcTable(resultCatalog, tbl.getTableName(), tbl.getTableType() diff --git a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Index.java b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Index.java index 72791b7..4e592db 100644 --- a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Index.java +++ b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Index.java @@ -1065,6 +1065,8 @@ public IndexStandardScheme getScheme() { public void read(org.apache.thrift.protocol.TProtocol iprot, Index struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); + String _key212; // required + String _val213; // required while (true) { schemeField = iprot.readFieldBegin(); @@ -1144,8 +1146,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, Index struct) throw struct.parameters = new HashMap(2*_map210.size); for (int _i211 = 0; _i211 < _map210.size; ++_i211) { - String _key212; // required - String _val213; // required _key212 = iprot.readString(); _val213 = iprot.readString(); struct.parameters.put(_key212, _val213); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 0dfbc5a..a1a345a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -177,15 +177,22 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, } // also the location field in partition List parts = msdb.getPartitions(dbname, name, -1); + String oldPartLoc; + Path oldPartLocPath; + String oldTblLocPath; + String newTblLocPath; + Path newPartLocPath; + URI oldUri; + String newPath; for (Partition part : parts) { - String oldPartLoc = part.getSd().getLocation(); - Path oldPartLocPath = new Path(oldPartLoc); - String oldTblLocPath = new Path(oldTblLoc).toUri().getPath(); - String newTblLocPath = new Path(newTblLoc).toUri().getPath(); + oldPartLoc = part.getSd().getLocation(); + oldPartLocPath = new Path(oldPartLoc); + oldTblLocPath = new Path(oldTblLoc).toUri().getPath(); + newTblLocPath = new Path(newTblLoc).toUri().getPath(); if (oldPartLoc.contains(oldTblLocPath)) { - Path newPartLocPath = null; - URI oldUri = oldPartLocPath.toUri(); - String newPath = oldUri.getPath().replace(oldTblLocPath, + newPartLocPath = null; + oldUri = oldPartLocPath.toUri(); + newPath = oldUri.getPath().replace(oldTblLocPath, newTblLocPath); newPartLocPath = new Path(oldUri.getScheme(), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java index 99e5b0f..51951d3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java @@ -170,8 +170,9 @@ public CommonJoinOperator(CommonJoinOperator clone) { Byte[] order, List[] aliasToObjectInspectors, T conf) { List structFieldObjectInspectors = new ArrayList(); + List oiList; for (Byte alias : order) { - List oiList = aliasToObjectInspectors[alias]; + oiList = aliasToObjectInspectors[alias]; if (oiList != null) { structFieldObjectInspectors.addAll(oiList); } @@ -233,8 +234,9 @@ protected void initializeOp(Configuration hconf) throws HiveException { rowContainerStandardObjectInspectors = joinValuesStandardObjectInspectors; } else { List[] rowContainerObjectInspectors = new List[tagLen]; + ArrayList rcOIs; for (Byte alias : order) { - ArrayList rcOIs = new ArrayList(); + rcOIs = new ArrayList(); rcOIs.addAll(joinValuesObjectInspectors[alias]); // for each alias, add object inspector for short as the last element rcOIs.add( @@ -258,9 +260,13 @@ protected void initializeOp(Configuration hconf) throws HiveException { // construct spill table serde which is used if input is too // large to fit into main memory. byte pos = 0; + ArrayList nr; + RowContainer> values; + RowContainer rc; + int sz; for (Byte alias : order) { - int sz = conf.getExprs().get(alias).size(); - ArrayList nr = new ArrayList(sz); + sz = conf.getExprs().get(alias).size(); + nr = new ArrayList(sz); for (int j = 0; j < sz; j++) { nr.add(null); @@ -274,7 +280,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { } dummyObj[pos] = nr; // there should be only 1 dummy object in the RowContainer - RowContainer> values = JoinUtil.getRowContainer(hconf, + values = JoinUtil.getRowContainer(hconf, rowContainerStandardObjectInspectors[pos], alias, 1, spillTableDesc, conf, !hasFilter(pos), reporter); @@ -283,7 +289,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { // if serde is null, the input doesn't need to be spilled out // e.g., the output columns does not contains the input table - RowContainer rc = JoinUtil.getRowContainer(hconf, + rc = JoinUtil.getRowContainer(hconf, rowContainerStandardObjectInspectors[pos], alias, joinCacheSize, spillTableDesc, conf, !hasFilter(pos), reporter); storage[pos] = rc; @@ -444,8 +450,9 @@ private void genJoinObject() throws HiveException { boolean rightFirst = true; boolean hasFilter = hasFilter(order[0]); AbstractRowContainer> aliasRes = storage[order[0]]; + boolean rightNull; for (List rightObj = aliasRes.first(); rightObj != null; rightObj = aliasRes.next()) { - boolean rightNull = rightObj == dummyObj[0]; + rightNull = rightObj == dummyObj[0]; if (hasFilter) { filterTags[0] = getFilterTag(rightObj); } @@ -478,11 +485,12 @@ private void genObject(int aliasNum, boolean allLeftFirst, boolean allLeftNull) boolean tryLOForFO = type == JoinDesc.FULL_OUTER_JOIN; boolean rightFirst = true; + boolean rightNull; for (List rightObj = aliasRes.first(); !done && rightObj != null; rightObj = loopAgain ? rightObj : aliasRes.next(), rightFirst = loopAgain = false) { System.arraycopy(prevSkip, 0, skip, 0, prevSkip.length); - boolean rightNull = rightObj == dummyObj[aliasNum]; + rightNull = rightObj == dummyObj[aliasNum]; if (hasFilter(order[aliasNum])) { filterTags[aliasNum] = getFilterTag(rightObj); } @@ -642,9 +650,10 @@ public void endGroup() throws HiveException { private void genUniqueJoinObject(int aliasNum, int forwardCachePos) throws HiveException { AbstractRowContainer> alias = storage[order[aliasNum]]; + int sz, p; for (ArrayList row = alias.first(); row != null; row = alias.next()) { - int sz = joinValues[order[aliasNum]].size(); - int p = forwardCachePos; + sz = joinValues[order[aliasNum]].size(); + p = forwardCachePos; for (int j = 0; j < sz; j++) { forwardCache[p++] = row.get(j); } @@ -660,9 +669,11 @@ private void genUniqueJoinObject(int aliasNum, int forwardCachePos) private void genAllOneUniqueJoinObject() throws HiveException { int p = 0; + ArrayList obj; + int sz; for (int i = 0; i < numAliases; i++) { - int sz = joinValues[order[i]].size(); - ArrayList obj = storage[order[i]].first(); + sz = joinValues[order[i]].size(); + obj = storage[order[i]].first(); for (int j = 0; j < sz; j++) { forwardCache[p++] = obj.get(j); } @@ -682,9 +693,11 @@ protected void checkAndGenObject() throws HiveException { // in a preserved table boolean hasNulls = false; // Will be true if there are null entries boolean allOne = true; + Byte alias; + AbstractRowContainer> alw; for (int i = 0; i < numAliases; i++) { - Byte alias = order[i]; - AbstractRowContainer> alw = storage[alias]; + alias = order[i]; + alw = storage[alias]; if (alw.size() != 1) { allOne = false; @@ -715,9 +728,11 @@ protected void checkAndGenObject() throws HiveException { // does any result need to be emitted boolean mayHasMoreThanOne = false; boolean hasEmpty = false; + Byte alias; + AbstractRowContainer> alw; for (int i = 0; i < numAliases; i++) { - Byte alias = order[i]; - AbstractRowContainer> alw = storage[alias]; + alias = order[i]; + alw = storage[alias]; if (noOuterJoin) { if (alw.size() == 0) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index ac3d686..1ae998b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -415,8 +415,9 @@ private void dpSetup() { ArrayList newFieldsOI = new ArrayList(); ArrayList newFieldsName = new ArrayList(); this.dpStartCol = 0; + String fn; for (StructField sf : fieldOI) { - String fn = sf.getFieldName(); + fn = sf.getFieldName(); if (!dpCtx.getInputToDPCols().containsKey(fn)) { newFieldsOI.add(sf.getFieldObjectInspector()); newFieldsName.add(sf.getFieldName()); @@ -637,8 +638,9 @@ public void processOp(Object row, int tag) throws HiveException { rowOutWriters[0].write(recordValue); } else { int keyHashCode = 0; + Object o; for (int i = 0; i < partitionEval.length; i++) { - Object o = partitionEval[i].evaluate(row); + o = partitionEval[i].evaluate(row); keyHashCode = keyHashCode * 31 + ObjectInspectorUtils.hashCode(o, partitionObjectInspectors[i]); } @@ -972,9 +974,11 @@ private void publishStats() throws HiveException { String taskID = Utilities.getTaskIdFromFilename(Utilities.getTaskId(hconf)); String spSpec = conf.getStaticSpec() != null ? conf.getStaticSpec() : ""; + FSPaths fspValue; + String key; + Map statsToPublish; for (String fspKey : valToPaths.keySet()) { - FSPaths fspValue = valToPaths.get(fspKey); - String key; + fspValue = valToPaths.get(fspKey); // construct the key(fileID) to insert into the intermediate stats table if (fspKey == "") { @@ -988,7 +992,7 @@ private void publishStats() throws HiveException { // common key prefix + static partition spec + DynamicPartSpec + taskID key = createKeyForStatsPublisher(taskID, spSpec, fspKey); } - Map statsToPublish = new HashMap(); + statsToPublish = new HashMap(); for (String statType : fspValue.stat.getStoredStats()) { statsToPublish.put(statType, Long.toString(fspValue.stat.getStat(statType))); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java index ccf55ba..e80bc2f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java @@ -214,9 +214,10 @@ protected void initializeOp(Configuration hconf) throws HiveException { newKeysGroupingSets = new ArrayList(); groupingSetsBitSet = new ArrayList(); + ExprNodeEvaluator groupingSetValueEvaluator; for (Integer groupingSet: groupingSets) { // Create the mapping corresponding to the grouping set - ExprNodeEvaluator groupingSetValueEvaluator = + groupingSetValueEvaluator = ExprNodeEvaluatorFactory.get(new ExprNodeConstantDesc(String.valueOf(groupingSet))); newKeysGroupingSets.add(groupingSetValueEvaluator.evaluate(null)); @@ -260,9 +261,15 @@ protected void initializeOp(Configuration hconf) throws HiveException { aggregationParameterStandardObjectInspectors = new ObjectInspector[aggrs.size()][]; aggregationParameterObjects = new Object[aggrs.size()][]; aggregationIsDistinct = new boolean[aggrs.size()]; + + AggregationDesc aggr; + ArrayList parameters; + String[] names; + String name; + Set set; for (int i = 0; i < aggrs.size(); i++) { - AggregationDesc aggr = aggrs.get(i); - ArrayList parameters = aggr.getParameters(); + aggr = aggrs.get(i); + parameters = aggr.getParameters(); aggregationParameterFields[i] = new ExprNodeEvaluator[parameters.size()]; aggregationParameterObjectInspectors[i] = new ObjectInspector[parameters .size()]; @@ -275,14 +282,14 @@ protected void initializeOp(Configuration hconf) throws HiveException { aggregationParameterObjectInspectors[i][j] = aggregationParameterFields[i][j] .initialize(rowInspector); if (unionExprEval != null) { - String[] names = parameters.get(j).getExprString().split("\\."); + names = parameters.get(j).getExprString().split("\\."); // parameters of the form : KEY.colx:t.coly if (Utilities.ReduceField.KEY.name().equals(names[0])) { - String name = names[names.length - 2]; + name = names[names.length - 2]; int tag = Integer.parseInt(name.split("\\:")[1]); if (aggr.getDistinct()) { // is distinct - Set set = distinctKeyAggrs.get(tag); + set = distinctKeyAggrs.get(tag); if (null == set) { set = new HashSet(); distinctKeyAggrs.put(tag, set); @@ -291,7 +298,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { set.add(i); } } else { - Set set = nonDistinctKeyAggrs.get(tag); + set = nonDistinctKeyAggrs.get(tag); if (null == set) { set = new HashSet(); nonDistinctKeyAggrs.put(tag, set); @@ -328,8 +335,10 @@ protected void initializeOp(Configuration hconf) throws HiveException { // init aggregationClasses aggregationEvaluators = new GenericUDAFEvaluator[conf.getAggregators() .size()]; + + AggregationDesc agg; for (int i = 0; i < aggregationEvaluators.length; i++) { - AggregationDesc agg = conf.getAggregators().get(i); + agg = conf.getAggregators().get(i); aggregationEvaluators[i] = agg.getGenericUDAFEvaluator(); } @@ -345,8 +354,10 @@ protected void initializeOp(Configuration hconf) throws HiveException { context.setup(genericUDAFEvaluator); } } + + ObjectInspector roi; for (int i = 0; i < aggregationEvaluators.length; i++) { - ObjectInspector roi = aggregationEvaluators[i].init(conf.getAggregators() + roi = aggregationEvaluators[i].init(conf.getAggregators() .get(i).getMode(), aggregationParameterObjectInspectors[i]); objectInspectors.add(roi); } @@ -598,9 +609,10 @@ protected void updateAggregations(AggregationBuffer[] aggs, Object row, ObjectInspector rowInspector, boolean hashAggr, boolean newEntryForHashAggr, Object[][] lastInvoke) throws HiveException { if (unionExprEval == null) { + Object[] o; for (int ai = 0; ai < aggs.length; ai++) { // Calculate the parameters - Object[] o = new Object[aggregationParameterFields[ai].length]; + o = new Object[aggregationParameterFields[ai].length]; for (int pi = 0; pi < aggregationParameterFields[ai].length; pi++) { o[pi] = aggregationParameterFields[ai][pi].evaluate(row); } @@ -640,8 +652,9 @@ protected void updateAggregations(AggregationBuffer[] aggs, Object row, // update non-distinct key aggregations : "KEY._colx:t._coly" if (nonDistinctKeyAggrs.get(unionTag) != null) { + Object[] o; for (int pos : nonDistinctKeyAggrs.get(unionTag)) { - Object[] o = new Object[aggregationParameterFields[pos].length]; + o = new Object[aggregationParameterFields[pos].length]; for (int pi = 0; pi < aggregationParameterFields[pos].length; pi++) { o[pi] = aggregationParameterFields[pos][pi].evaluate(row); } @@ -651,8 +664,9 @@ protected void updateAggregations(AggregationBuffer[] aggs, Object row, // there may be multi distinct clauses for one column // update them all. if (distinctKeyAggrs.get(unionTag) != null) { + Object[] o; for (int i : distinctKeyAggrs.get(unionTag)) { - Object[] o = new Object[aggregationParameterFields[i].length]; + o = new Object[aggregationParameterFields[i].length]; for (int pi = 0; pi < aggregationParameterFields[i].length; pi++) { o[pi] = aggregationParameterFields[i][pi].evaluate(row); } @@ -683,8 +697,9 @@ protected void updateAggregations(AggregationBuffer[] aggs, Object row, // update non-distinct value aggregations: 'VALUE._colx' // these aggregations should be updated only once. if (unionTag == 0) { + Object[] o; for (int pos : nonDistinctAggrs) { - Object[] o = new Object[aggregationParameterFields[pos].length]; + o = new Object[aggregationParameterFields[pos].length]; for (int pi = 0; pi < aggregationParameterFields[pos].length; pi++) { o[pi] = aggregationParameterFields[pos][pi].evaluate(row); } @@ -692,10 +707,11 @@ protected void updateAggregations(AggregationBuffer[] aggs, Object row, } } } else { + Object[] o; for (int ai = 0; ai < aggs.length; ai++) { // there is no distinct aggregation, // update all aggregations - Object[] o = new Object[aggregationParameterFields[ai].length]; + o = new Object[aggregationParameterFields[ai].length]; for (int pi = 0; pi < aggregationParameterFields[ai].length; pi++) { o[pi] = aggregationParameterFields[ai][pi].evaluate(row); } @@ -914,8 +930,9 @@ private boolean shouldBeFlushed(KeyWrapper newKeys) { if(rate > memoryThreshold){ return true; } + Object key; for (Integer pos : keyPositionsSize) { - Object key = newKeys.getKeyArray()[pos.intValue()]; + key = newKeys.getKeyArray()[pos.intValue()]; // Ignore nulls if (key != null) { if (key instanceof LazyString) { @@ -939,8 +956,9 @@ private boolean shouldBeFlushed(KeyWrapper newKeys) { } AggregationBuffer[] aggs = hashAggregations.get(newKeys); + AggregationBuffer agg; for (int i = 0; i < aggs.length; i++) { - AggregationBuffer agg = aggs[i]; + agg = aggs[i]; if (GenericUDAFEvaluator.isEstimable(agg)) { totalVariableSize += ((GenericUDAFEvaluator.AbstractAggregationBuffer)agg).estimate(); continue; @@ -968,9 +986,10 @@ private boolean shouldBeFlushed(KeyWrapper newKeys) { private int estimateSize(AggregationBuffer agg, List fields) { int length = 0; + Object o; for (Field f : fields) { try { - Object o = f.get(agg); + o = f.get(agg); if (o instanceof String){ length += ((String)o).length(); } @@ -999,8 +1018,9 @@ private void flushHashTable(boolean complete) throws HiveException { if (complete) { Iterator> iter = hashAggregations .entrySet().iterator(); + Map.Entry m; while (iter.hasNext()) { - Map.Entry m = iter.next(); + m = iter.next(); forward(m.getKey().getKeyArray(), m.getValue()); } hashAggregations.clear(); @@ -1014,8 +1034,9 @@ private void flushHashTable(boolean complete) throws HiveException { Iterator> iter = hashAggregations .entrySet().iterator(); int numDel = 0; + Map.Entry m; while (iter.hasNext()) { - Map.Entry m = iter.next(); + m = iter.next(); forward(m.getKey().getKeyArray(), m.getValue()); iter.remove(); numDel++; @@ -1064,8 +1085,9 @@ public void flush() throws HiveException{ LOG.info("Begin Hash Table flush: size = " + hashAggregations.size()); Iterator iter = hashAggregations.entrySet().iterator(); + Map.Entry m; while (iter.hasNext()) { - Map.Entry m = (Map.Entry) iter + m = (Map.Entry) iter .next(); forward(m.getKey().getKeyArray(), m.getValue()); @@ -1155,8 +1177,9 @@ public void closeOp(boolean abort) throws HiveException { } ArrayList aggrs = conf.getAggregators(); + ArrayList params; for (AggregationDesc aggr : aggrs) { - ArrayList params = aggr.getParameters(); + params = aggr.getParameters(); for (ExprNodeDesc param : params) { colLists = Utilities.mergeUniqElems(colLists, param.getCols()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java index fb6ecb4..001e371 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java @@ -233,11 +233,12 @@ protected void initializeOp(Configuration hconf) throws HiveException { rowContainerStandardObjectInspectors = joinValuesStandardObjectInspectors; } else { List[] rowContainerObjectInspectors = new List[tagLen]; + List rcOIs; for (Byte alias : order) { if (alias == posBigTableAlias) { continue; } - List rcOIs = joinValuesObjectInspectors[alias]; + rcOIs = joinValuesObjectInspectors[alias]; if (filterMaps != null && filterMaps[alias] != null) { // for each alias, add object inspector for filter tag as the last element rcOIs = new ArrayList(rcOIs); @@ -266,12 +267,13 @@ protected void initializeOp(Configuration hconf) throws HiveException { } // initialize the hash tables for other tables + HashMapWrapper hashTable; for (Byte pos : order) { if (pos == posBigTableAlias) { continue; } - HashMapWrapper hashTable = new HashMapWrapper( + hashTable = new HashMapWrapper( hashTableThreshold, hashTableLoadFactor, hashTableMaxMemoryUsage); mapJoinTables[pos] = hashTable; @@ -283,12 +285,14 @@ protected void initializeOp(Configuration hconf) throws HiveException { protected static List[] getStandardObjectInspectors( List[] aliasToObjectInspectors, int maxTag) { List[] result = new List[maxTag]; + List oiList; + ArrayList fieldOIList; for (byte alias = 0; alias < aliasToObjectInspectors.length; alias++) { - List oiList = aliasToObjectInspectors[alias]; + oiList = aliasToObjectInspectors[alias]; if (oiList == null) { continue; } - ArrayList fieldOIList = new ArrayList(oiList.size()); + fieldOIList = new ArrayList(oiList.size()); for (int i = 0; i < oiList.size(); i++) { fieldOIList.add(ObjectInspectorUtils.getStandardObjectInspector(oiList.get(i), ObjectInspectorCopyOption.WRITABLE)); @@ -387,8 +391,9 @@ private void setValueMetaData(int tag) throws SerDeException { List newFields = rowContainerStandardObjectInspectors[alias]; int length = newFields.size(); List newNames = new ArrayList(length); + String tmp; for (int i = 0; i < length; i++) { - String tmp = new String("tmp_" + i); + tmp = new String("tmp_" + i); newNames.add(tmp); } StandardStructObjectInspector standardOI = ObjectInspectorFactory @@ -407,27 +412,32 @@ public void closeOp(boolean abort) throws HiveException { String tmpURI = this.getExecContext().getLocalWork().getTmpFileURI(); LOG.info("Get TMP URI: " + tmpURI); long fileLength; + HashMapWrapper hashTable; + String bigBucketFileName, fileName, dumpFilePrefix, tmpURIPath; + Path path; + FileSystem fs; + File file; for (byte tag = 0; tag < mapJoinTables.length; tag++) { // get the key and value - HashMapWrapper hashTable = mapJoinTables[tag]; + hashTable = mapJoinTables[tag]; if (hashTable == null) { continue; } // get current input file name - String bigBucketFileName = getExecContext().getCurrentBigBucketFile(); + bigBucketFileName = getExecContext().getCurrentBigBucketFile(); - String fileName = getExecContext().getLocalWork().getBucketFileName(bigBucketFileName); + fileName = getExecContext().getLocalWork().getBucketFileName(bigBucketFileName); // get the tmp URI path; it will be a hdfs path if not local mode - String dumpFilePrefix = conf.getDumpFilePrefix(); - String tmpURIPath = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName); + dumpFilePrefix = conf.getDumpFilePrefix(); + tmpURIPath = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName); hashTable.isAbort(rowNumber, console); console.printInfo(Utilities.now() + "\tDump the hashtable into file: " + tmpURIPath); // get the hashtable file and path - Path path = new Path(tmpURIPath); - FileSystem fs = path.getFileSystem(hconf); - File file = new File(path.toUri().getPath()); + path = new Path(tmpURIPath); + fs = path.getFileSystem(hconf); + file = new File(path.toUri().getPath()); fs.create(path); fileLength = hashTable.flushMemoryCacheToPersistent(file); console.printInfo(Utilities.now() + "\tUpload 1 File to: " + tmpURIPath + " File size: " diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java index b22b54c..7d61099 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -150,8 +151,9 @@ public void jobCloseOp(Configuration hconf, boolean success, JobCloseFeedBack fe int numAliases = conf.getExprs().size(); if (conf.getHandleSkewJoin()) { try { + String specPath; for (int i = 0; i < numAliases; i++) { - String specPath = conf.getBigKeysDirMap().get((byte) i); + specPath = conf.getBigKeysDirMap().get((byte) i); mvFileToFinalPath(specPath, hconf, success, LOG); for (int j = 0; j < numAliases; j++) { if (j == i) { @@ -162,11 +164,11 @@ public void jobCloseOp(Configuration hconf, boolean success, JobCloseFeedBack fe mvFileToFinalPath(specPath, hconf, success, LOG); } } - + if (success) { // move up files for (int i = 0; i < numAliases; i++) { - String specPath = conf.getBigKeysDirMap().get((byte) i); + specPath = conf.getBigKeysDirMap().get((byte) i); moveUpFiles(specPath, hconf, LOG); for (int j = 0; j < numAliases; j++) { if (j == i) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java index 31dbf41..5c96239 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java @@ -55,6 +55,8 @@ ObjectInspector[] inputObjInspector, int posBigTableAlias, int tagLen) throws HiveException { List[] result = new List[tagLen]; + List exprList; + List fieldOIList; for (byte alias = 0; alias < exprEntries.length; alias++) { //get big table if (alias == (byte) posBigTableAlias){ @@ -62,8 +64,8 @@ continue; } - List exprList = exprEntries[alias]; - List fieldOIList = new ArrayList(); + exprList = exprEntries[alias]; + fieldOIList = new ArrayList(); for (int i = 0; i < exprList.size(); i++) { fieldOIList.add(exprList.get(i).initialize(inputObjInspector[alias])); } @@ -77,6 +79,8 @@ List[] aliasToObjectInspectors, int posBigTableAlias, int tagLen) { List[] result = new List[tagLen]; + List oiList; + ArrayList fieldOIList; for (byte alias = 0; alias < aliasToObjectInspectors.length; alias++) { //get big table if(alias == (byte) posBigTableAlias ){ @@ -84,8 +88,8 @@ continue; } - List oiList = aliasToObjectInspectors[alias]; - ArrayList fieldOIList = new ArrayList( + oiList = aliasToObjectInspectors[alias]; + fieldOIList = new ArrayList( oiList.size()); for (int i = 0; i < oiList.size(); i++) { fieldOIList.add(ObjectInspectorUtils.getStandardObjectInspector(oiList @@ -107,9 +111,11 @@ public static int populateJoinKeyValue(List[] outMap, Byte[] order, int posBigTableAlias) throws HiveException { int total = 0; + Byte key; + List valueFields; for (Entry> e : inputMap.entrySet()) { - Byte key = order == null ? e.getKey() : order[e.getKey()]; - List valueFields = new ArrayList(); + key = order == null ? e.getKey() : order[e.getKey()]; + valueFields = new ArrayList(); for (ExprNodeDesc expr : e.getValue()) { if (key == (byte) posBigTableAlias) { valueFields.add(null); @@ -260,10 +266,12 @@ protected static short isFiltered(Object row, List filters, int length = filterMap[i + 1]; boolean passed = true; + Object condition; + Boolean result; for (; length > 0; length--, j++) { if (passed) { - Object condition = filters.get(j).evaluate(row); - Boolean result = (Boolean) ((PrimitiveObjectInspector) + condition = filters.get(j).evaluate(row); + result = (Boolean) ((PrimitiveObjectInspector) ois.get(j)).getPrimitiveJavaObject(condition); if (result == null || !result) { passed = false; @@ -314,16 +322,21 @@ public static SerDe getSpillSerDe(byte alias, TableDesc[] spillTableDesc, int tagLen = conf.getTagLength(); Map> exprs = conf.getExprs(); TableDesc[] spillTableDesc = new TableDesc[tagLen]; + List valueCols; + StringBuilder colNames, colTypes; + int columnSize; + String newColName; + TableDesc tblDesc; for (int tag = 0; tag < exprs.size(); tag++) { - List valueCols = exprs.get((byte) tag); - int columnSize = valueCols.size(); - StringBuilder colNames = new StringBuilder(); - StringBuilder colTypes = new StringBuilder(); + valueCols = exprs.get((byte) tag); + columnSize = valueCols.size(); + colNames = new StringBuilder(); + colTypes = new StringBuilder(); if (columnSize <= 0) { continue; } for (int k = 0; k < columnSize; k++) { - String newColName = tag + "_VALUE_" + k; // any name, it does not + newColName = tag + "_VALUE_" + k; // any name, it does not // matter. colNames.append(newColName); colNames.append(','); @@ -339,7 +352,7 @@ public static SerDe getSpillSerDe(byte alias, TableDesc[] spillTableDesc, // remove the last ',' colNames.setLength(colNames.length() - 1); colTypes.setLength(colTypes.length() - 1); - TableDesc tblDesc = new TableDesc(LazyBinarySerDe.class, + tblDesc = new TableDesc(LazyBinarySerDe.class, SequenceFileInputFormat.class, HiveSequenceFileOutputFormat.class, Utilities.makeProperties( org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "" diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java index b5c85d2..225c187 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java @@ -547,15 +547,18 @@ public void process(Writable value) throws HiveException { if (vcValues == null) { vcValues = new Writable[vcs.size()]; } + VirtualColumn vc; + LongWritable old; + long current; for (int i = 0; i < vcs.size(); i++) { - VirtualColumn vc = vcs.get(i); + vc = vcs.get(i); if (vc.equals(VirtualColumn.FILENAME)) { if (ctx.inputFileChanged()) { vcValues[i] = new Text(ctx.getCurrentInputFile()); } } else if (vc.equals(VirtualColumn.BLOCKOFFSET)) { - long current = ctx.getIoCxt().getCurrentBlockStart(); - LongWritable old = (LongWritable) vcValues[i]; + current = ctx.getIoCxt().getCurrentBlockStart(); + old = (LongWritable) vcValues[i]; if (old == null) { old = new LongWritable(current); vcValues[i] = old; @@ -565,8 +568,8 @@ public void process(Writable value) throws HiveException { old.set(current); } } else if (vc.equals(VirtualColumn.ROWOFFSET)) { - long current = ctx.getIoCxt().getCurrentRow(); - LongWritable old = (LongWritable) vcValues[i]; + current = ctx.getIoCxt().getCurrentRow(); + old = (LongWritable) vcValues[i]; if (old == null) { old = new LongWritable(current); vcValues[i] = old; @@ -576,12 +579,12 @@ public void process(Writable value) throws HiveException { old.set(current); } } else if (vc.equals(VirtualColumn.RAWDATASIZE)) { - long current = 0L; + current = 0L; SerDeStats stats = deserializer.getSerDeStats(); if(stats != null) { current = stats.getRawDataSize(); } - LongWritable old = (LongWritable) vcValues[i]; + old = (LongWritable) vcValues[i]; if (old == null) { old = new LongWritable(current); vcValues[i] = old; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java index 458d259..356de68 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java @@ -350,8 +350,9 @@ public void initialize(Configuration hconf, ObjectInspector[] inputOIs) childOperatorsArray[i] = childOperators.get(i); } childOperatorsTag = new int[childOperatorsArray.length]; + List> parentOperators; for (int i = 0; i < childOperatorsArray.length; i++) { - List> parentOperators = childOperatorsArray[i] + parentOperators = childOperatorsArray[i] .getParentOperators(); if (parentOperators == null) { throw new HiveException("Hive internal error: parent is null in " @@ -380,8 +381,9 @@ public void initialize(Configuration hconf, ObjectInspector[] inputOIs) public void initializeLocalWork(Configuration hconf) throws HiveException { if (childOperators != null) { + Operator childOp; for (int i =0; i childOp = this.childOperators.get(i); + childOp = this.childOperators.get(i); childOp.initializeLocalWork(hconf); } } @@ -725,9 +727,11 @@ public void removeChildAndAdoptItsChildren( childOperators.addAll(childIndex, child.getChildOperators()); } + List> parents; + int index; for (Operator gc : child.getChildOperators()) { - List> parents = gc.getParentOperators(); - int index = parents.indexOf(child); + parents = gc.getParentOperators(); + index = parents.indexOf(child); if (index == -1) { throw new SemanticException( "Exception when trying to remove partition predicates: fail to find parent from child"); @@ -840,8 +844,9 @@ protected void forward(Object row, ObjectInspector rowInspector) } int childrenDone = 0; + Operator o; for (int i = 0; i < childOperatorsArray.length; i++) { - Operator o = childOperatorsArray[i]; + o = childOperatorsArray[i]; if (o.getDone()) { childrenDone++; } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index edb55fa..13fd5f9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -1527,11 +1527,12 @@ public static void removeTempOrDuplicateFiles(FileSystem fs, Path path) throws I // get the missing buckets and generate empty buckets String taskID1 = taskIDToFile.keySet().iterator().next(); Path bucketPath = taskIDToFile.values().iterator().next().getPath(); - for (int j = 0; j < dpCtx.getNumBuckets(); ++j) { - String taskID2 = replaceTaskId(taskID1, j); + String taskID2, path2; + for (int j = 0; j < dpCtx.getNumBuckets(); ++j) { + taskID2 = replaceTaskId(taskID1, j); if (!taskIDToFile.containsKey(taskID2)) { // create empty bucket, file name should be derived from taskID2 - String path2 = replaceTaskIdFromFilename(bucketPath.toUri().getPath().toString(), j); + path2 = replaceTaskIdFromFilename(bucketPath.toUri().getPath().toString(), j); result.add(path2); } } @@ -1729,12 +1730,15 @@ public static String formatBinaryString(byte[] array, int start, int length) { public static void validateColumnNames(List colNames, List checkCols) throws SemanticException { Iterator checkColsIter = checkCols.iterator(); - while (checkColsIter.hasNext()) { - String toCheck = checkColsIter.next(); - boolean found = false; - Iterator colNamesIter = colNames.iterator(); + String toCheck, colName; + boolean found; + Iterator colNamesIter; + while (checkColsIter.hasNext()) { + toCheck = checkColsIter.next(); + found = false; + colNamesIter = colNames.iterator(); while (colNamesIter.hasNext()) { - String colName = colNamesIter.next(); + colName = colNamesIter.next(); if (toCheck.equalsIgnoreCase(colName)) { found = true; break; @@ -1811,14 +1815,16 @@ public static ContentSummary getInputSummary(Context ctx, MapWork work, PathFilt // this method will avoid number of threads out of control. synchronized (INPUT_SUMMARY_LOCK) { // For each input path, calculate the total size. + Path pa; + ContentSummary cs; for (String path : work.getPathToAliases().keySet()) { - Path p = new Path(path); + pa = new Path(path); - if (filter != null && !filter.accept(p)) { + if (filter != null && !filter.accept(pa)) { continue; } - ContentSummary cs = ctx.getCS(path); + cs = ctx.getCS(path); if (cs == null) { if (path == null) { continue; @@ -1833,7 +1839,7 @@ public static ContentSummary getInputSummary(Context ctx, MapWork work, PathFilt // Process the case when name node call is needed final Map resultMap = new ConcurrentHashMap(); - ArrayList> results = new ArrayList>(); + ArrayList>results = new ArrayList>(); final ThreadPoolExecutor executor; int maxThreads = ctx.getConf().getInt("mapred.dfsclient.parallelism.max", 0); if (pathNeedProcess.size() > 1 && maxThreads > 1) { @@ -1906,8 +1912,9 @@ public void run() { } if (executor != null) { + boolean executorDone; for (Future result : results) { - boolean executorDone = false; + executorDone = false; do { try { result.get(); @@ -1925,7 +1932,7 @@ public void run() { } HiveInterruptUtils.checkInterrupted(); for (Map.Entry entry : resultMap.entrySet()) { - ContentSummary cs = entry.getValue(); + cs = entry.getValue(); summary[0] += cs.getLength(); summary[1] += cs.getFileCount(); @@ -2015,14 +2022,16 @@ private static void getMRTasks(List> tasks, List fullPartSpec; for (int i = 0; i < status.length; ++i) { // get the dynamically created directory - Path partPath = status[i].getPath(); + partPath = status[i].getPath(); assert fs.getFileStatus(partPath).isDir() : "partitions " + partPath + " is not a directory !"; // generate a full partition specification - LinkedHashMap fullPartSpec = new LinkedHashMap(partSpec); + fullPartSpec = new LinkedHashMap(partSpec); Warehouse.makeSpecFromName(fullPartSpec, partPath); fullPartSpecs.add(fullPartSpec); } @@ -2070,13 +2079,14 @@ public static void setColumnNameList(JobConf jobConf, Operator op) { return; } StringBuilder columnNames = new StringBuilder(); + String columnNamesString; for (ColumnInfo colInfo : rowSchema.getSignature()) { if (columnNames.length() > 0) { columnNames.append(","); } columnNames.append(colInfo.getInternalName()); } - String columnNamesString = columnNames.toString(); + columnNamesString = columnNames.toString(); jobConf.set(serdeConstants.LIST_COLUMNS, columnNamesString); } @@ -2086,13 +2096,14 @@ public static void setColumnTypeList(JobConf jobConf, Operator op) { return; } StringBuilder columnTypes = new StringBuilder(); + String columnTypesString; for (ColumnInfo colInfo : rowSchema.getSignature()) { if (columnTypes.length() > 0) { columnTypes.append(","); } columnTypes.append(colInfo.getType().getTypeName()); } - String columnTypesString = columnTypes.toString(); + columnTypesString = columnTypes.toString(); jobConf.set(serdeConstants.LIST_COLUMN_TYPES, columnTypesString); } @@ -2303,8 +2314,9 @@ public static void reworkMapRedWork(Task task, try { MapredWork mapredWork = ((MapRedTask) task).getWork(); Set> reworkInputFormats = new HashSet>(); + Class inputFormatCls; for (PartitionDesc part : mapredWork.getMapWork().getPathToPartitionInfo().values()) { - Class inputFormatCls = part + inputFormatCls = part .getInputFileFormatClass(); if (ReworkMapredInputFormat.class.isAssignableFrom(inputFormatCls)) { reworkInputFormats.add(inputFormatCls); @@ -2312,9 +2324,10 @@ public static void reworkMapRedWork(Task task, } if (reworkInputFormats.size() > 0) { - for (Class inputFormatCls : reworkInputFormats) { - ReworkMapredInputFormat inst = (ReworkMapredInputFormat) ReflectionUtils - .newInstance(inputFormatCls, null); + ReworkMapredInputFormat inst; + for (Class inputFormatCls2 : reworkInputFormats) { + inst = (ReworkMapredInputFormat) ReflectionUtils + .newInstance(inputFormatCls2, null); inst.rework(conf, mapredWork); } } @@ -2393,9 +2406,10 @@ public static Connection connectWithRetry(String connectionString, Random r = new Random(); // retry with # of maxRetries before throwing exception + Connection conn; for (int failures = 0; ; failures++) { try { - Connection conn = DriverManager.getConnection(connectionString); + conn = DriverManager.getConnection(connectionString); return conn; } catch (SQLTransientException e) { if (failures >= maxRetries) { @@ -2650,9 +2664,10 @@ public static long getTotalInputNumFiles (ContentSummary inputSummary, MapWork w */ public static double getHighestSamplePercentage (MapWork work) { double highestSamplePercentage = 0; + Double rate; for (String alias : work.getAliasToWork().keySet()) { if (work.getNameToSplitSample().containsKey(alias)) { - Double rate = work.getNameToSplitSample().get(alias).getPercent(); + rate = work.getNameToSplitSample().get(alias).getPercent(); if (rate != null && rate > highestSamplePercentage) { highestSamplePercentage = rate; } @@ -2686,11 +2701,12 @@ public static double getHighestSamplePercentage (MapWork work) { Set pathsProcessed = new HashSet(); List pathsToAdd = new LinkedList(); // AliasToWork contains all the aliases + Path path; for (String alias : work.getAliasToWork().keySet()) { LOG.info("Processing alias " + alias); // The alias may not have any path - Path path = null; + path = null; for (String file : new LinkedList(work.getPathToAliases().keySet())) { List aliases = work.getPathToAliases().get(file); if (aliases.contains(alias)) { @@ -2929,17 +2945,21 @@ public static void createTmpDirs(Configuration conf, ReduceWork rWork) private static void createTmpDirs(Configuration conf, List> ops) throws IOException { - + Operator op; + FileSinkDesc fdesc; + String tempDir; + Path tempPath; + FileSystem fs; while (!ops.isEmpty()) { - Operator op = ops.remove(0); + op = ops.remove(0); if (op instanceof FileSinkOperator) { - FileSinkDesc fdesc = ((FileSinkOperator) op).getConf(); - String tempDir = fdesc.getDirName(); + fdesc = ((FileSinkOperator) op).getConf(); + tempDir = fdesc.getDirName(); if (tempDir != null) { - Path tempPath = Utilities.toTempPath(new Path(tempDir)); - FileSystem fs = tempPath.getFileSystem(conf); + tempPath = Utilities.toTempPath(new Path(tempDir)); + fs = tempPath.getFileSystem(conf); fs.mkdirs(tempPath); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java index d1e82a2..3529a3e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MapredLocalWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MapReduceBase; @@ -217,8 +218,9 @@ public void close() { if (fetchOperators != null) { MapredLocalWork localWork = mo.getConf().getMapLocalWork(); + Operator forwardOp; for (Map.Entry entry : fetchOperators.entrySet()) { - Operator forwardOp = localWork + forwardOp = localWork .getAliasToWork().get(entry.getKey()); forwardOp.close(abort); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/JobDebugger.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/JobDebugger.java index 7b77944..b517556 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/JobDebugger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/JobDebugger.java @@ -124,8 +124,9 @@ public void run() { public static int extractErrorCode(String[] diagnostics) { int result = 0; Pattern errorCodeRegex = ErrorMsg.getErrorCodePattern(); + Matcher matcher; for (String mesg : diagnostics) { - Matcher matcher = errorCodeRegex.matcher(mesg); + matcher = errorCodeRegex.matcher(mesg); if (matcher.find()) { result = Integer.parseInt(matcher.group(1)); // We don't exit the loop early because we want to extract the error code @@ -147,6 +148,7 @@ public void run() { private void getTaskInfos() throws IOException, MalformedURLException { int startIndex = 0; + String[] taskJobIds; while (true) { TaskCompletionEvent[] taskCompletions = rj.getTaskCompletionEvents(startIndex); @@ -156,10 +158,13 @@ private void getTaskInfos() throws IOException, MalformedURLException { boolean more = true; boolean firstError = true; + String taskId, jobId; + TaskInfo ti; + String taskAttemptLogUrl; for (TaskCompletionEvent t : taskCompletions) { // getTaskJobIDs returns Strings for compatibility with Hadoop versions // without TaskID or TaskAttemptID - String[] taskJobIds = ShimLoader.getHadoopShims().getTaskJobIDs(t); + taskJobIds = ShimLoader.getHadoopShims().getTaskJobIDs(t); if (taskJobIds == null) { console.printError("Task attempt info is unavailable in this Hadoop version"); @@ -169,21 +174,21 @@ private void getTaskInfos() throws IOException, MalformedURLException { // For each task completion event, get the associated task id, job id // and the logs - String taskId = taskJobIds[0]; - String jobId = taskJobIds[1]; + taskId = taskJobIds[0]; + jobId = taskJobIds[1]; if (firstError) { console.printError("Examining task ID: " + taskId + " (and more) from job " + jobId); firstError = false; } - TaskInfo ti = taskIdToInfo.get(taskId); + ti = taskIdToInfo.get(taskId); if (ti == null) { ti = new TaskInfo(jobId); taskIdToInfo.put(taskId, ti); } // These tasks should have come from the same job. assert (ti.getJobId() != null && ti.getJobId().equals(jobId)); - String taskAttemptLogUrl = ShimLoader.getHadoopShims().getTaskAttemptLogUrl( + taskAttemptLogUrl = ShimLoader.getHadoopShims().getTaskAttemptLogUrl( conf, t.getTaskTrackerHttp(), t.getTaskId()); if (taskAttemptLogUrl != null) { ti.getLogUrls().add(taskAttemptLogUrl); @@ -265,14 +270,18 @@ private void showJobFailDebugInfo() throws IOException { + e.getMessage()); } + TaskInfo ti; + String jobId; + String taskUrl; + TaskLogProcessor tlp; for (String task : failures.keySet()) { if (failures.get(task).intValue() == maxFailures) { - TaskInfo ti = taskIdToInfo.get(task); - String jobId = ti.getJobId(); - String taskUrl = (jtUrl == null) ? null : + ti = taskIdToInfo.get(task); + jobId = ti.getJobId(); + taskUrl = (jtUrl == null) ? null : jtUrl + "/taskdetails.jsp?jobid=" + jobId + "&tipid=" + task.toString(); - TaskLogProcessor tlp = new TaskLogProcessor(conf); + tlp = new TaskLogProcessor(conf); for (String logUrl : ti.getLogUrls()) { tlp.addTaskAttemptLogUrl(logUrl); } @@ -324,9 +333,10 @@ private void showJobFailDebugInfo() throws IOException { } public int getErrorCode() { + TaskInfo ti; for (String task : failures.keySet()) { if (failures.get(task).intValue() == maxFailures) { - TaskInfo ti = taskIdToInfo.get(task); + ti = taskIdToInfo.get(task); return ti.getErrorCode(); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java b/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java index 332ced8..d93f2b9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexResult.java @@ -93,12 +93,16 @@ public HiveIndexResult(List indexFiles, JobConf conf) throws IOException if (indexFiles != null && indexFiles.size() > 0) { List paths = new ArrayList(); + Path indexFilePath; + FileSystem fs; + FileStatus indexStat; + FileStatus[] fss; for (String indexFile : indexFiles) { - Path indexFilePath = new Path(indexFile); - FileSystem fs = indexFilePath.getFileSystem(conf); - FileStatus indexStat = fs.getFileStatus(indexFilePath); + indexFilePath = new Path(indexFile); + fs = indexFilePath.getFileSystem(conf); + indexStat = fs.getFileStatus(indexFilePath); if (indexStat.isDir()) { - FileStatus[] fss = fs.listStatus(indexFilePath); + fss = fs.listStatus(indexFilePath); for (FileStatus f : fss) { paths.add(f.getPath()); } @@ -113,12 +117,15 @@ public HiveIndexResult(List indexFiles, JobConf conf) throws IOException } long lineCounter = 0; + FSDataInputStream ifile; + LineReader lr; + Text line; for (Path indexFinalPath : paths) { - FileSystem fs = indexFinalPath.getFileSystem(conf); - FSDataInputStream ifile = fs.open(indexFinalPath); - LineReader lr = new LineReader(ifile, conf); + fs = indexFinalPath.getFileSystem(conf); + ifile = fs.open(indexFinalPath); + lr = new LineReader(ifile, conf); try { - Text line = new Text(); + line = new Text(); while (lr.readLine(line) > 0) { if (++lineCounter > maxEntriesToLoad) { throw new HiveException("Number of compact index entries loaded during the query exceeded the maximum of " + maxEntriesToLoad diff --git a/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java index c6c03d0..dd0c284 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java @@ -289,12 +289,13 @@ public ExprNodeDesc translateSearchConditions( List searchConditions) { ExprNodeDesc expr = null; + List children; for (IndexSearchCondition searchCondition : searchConditions) { if (expr == null) { expr = searchCondition.getComparisonExpr(); continue; } - List children = new ArrayList(); + children = new ArrayList(); children.add(expr); children.add(searchCondition.getComparisonExpr()); expr = new ExprNodeGenericFuncDesc( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java index 4a77645..eb0d83f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/index/TableBasedIndexHandler.java @@ -71,9 +71,11 @@ // check whether the index table partitions are still exists in base // table + Partition indexPart, basePart; + Task indexBuilder; for (int i = 0; i < indexTblPartitions.size(); i++) { - Partition indexPart = indexTblPartitions.get(i); - Partition basePart = null; + indexPart = indexTblPartitions.get(i); + basePart = null; for (int j = 0; j < baseTblPartitions.size(); j++) { if (baseTblPartitions.get(j).getName().equals(indexPart.getName())) { basePart = baseTblPartitions.get(j); @@ -86,7 +88,7 @@ "Partitions of base table and index table are inconsistent."); } // for each partition, spawn a map reduce task. - Task indexBuilder = getIndexBuilderMapRedTask(inputs, outputs, index.getSd().getCols(), true, + indexBuilder = getIndexBuilderMapRedTask(inputs, outputs, index.getSd().getCols(), true, new PartitionDesc(indexPart), indexTbl.getTableName(), new PartitionDesc(basePart), baseTbl.getTableName(), indexTbl.getDbName()); indexBuilderTasks.add(indexBuilder); @@ -107,9 +109,11 @@ LinkedHashMap partSpec) { List ret = new ArrayList(partSpec.size()); Iterator> iter = partSpec.entrySet().iterator(); + StringBuilder sb; + Entry p; while (iter.hasNext()) { - StringBuilder sb = new StringBuilder(); - Entry p = iter.next(); + sb = new StringBuilder(); + p = iter.next(); sb.append(HiveUtils.unparseIdentifier(p.getKey())); sb.append(" = "); sb.append("'"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputFormat.java index 49145b7..817511a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/BucketizedHiveInputFormat.java @@ -128,11 +128,12 @@ public RecordReader getRecordReader(InputSplit split, JobConf job, FileStatus[] listStatus = listStatus(newjob, dir); + InputSplit[] iss; for (FileStatus status : listStatus) { LOG.info("block size: " + status.getBlockSize()); LOG.info("file length: " + status.getLen()); FileInputFormat.setInputPaths(newjob, status.getPath()); - InputSplit[] iss = inputFormat.getSplits(newjob, 0); + iss = inputFormat.getSplits(newjob, 0); if (iss != null && iss.length > 0) { numOrigSplits += iss.length; result.add(new BucketizedHiveInputSplit(iss, inputFormatClass diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java index b9914dc..d06d45f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java @@ -288,20 +288,33 @@ public int hashCode() { new HashMap(); Set poolSet = new HashSet(); + PartitionDesc part; + TableDesc tableDesc; + Class inputFormatClass; + String inputFormatClassName; + InputFormat inputFormat; + String deserializerClassName; + FileSystem inpFs; + Queue dirs; + FileStatus fStats; + Path tstPath; + FileStatus[] fStatus; + Path filterPath; + CombineFilter f; + List> opList; for (Path path : paths) { - - PartitionDesc part = HiveFileFormatUtils.getPartitionDescFromPathRecursively( + part = HiveFileFormatUtils.getPartitionDescFromPathRecursively( pathToPartitionInfo, path, IOPrepareCache.get().allocatePartitionDescMap()); - TableDesc tableDesc = part.getTableDesc(); + tableDesc = part.getTableDesc(); if ((tableDesc != null) && tableDesc.isNonNative()) { return super.getSplits(job, numSplits); } // Use HiveInputFormat if any of the paths is not splittable - Class inputFormatClass = part.getInputFileFormatClass(); - String inputFormatClassName = inputFormatClass.getName(); - InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job); - String deserializerClassName = part.getDeserializerClass() == null ? null + inputFormatClass = part.getInputFileFormatClass(); + inputFormatClassName = inputFormatClass.getName(); + inputFormat = getInputFormatFromCache(inputFormatClass, job); + deserializerClassName = part.getDeserializerClass() == null ? null : part.getDeserializerClass().getName(); // Since there is no easy way of knowing whether MAPREDUCE-1597 is present in the tree or not, @@ -311,11 +324,11 @@ public int hashCode() { // https://issues.apache.org/jira/browse/MAPREDUCE-1597 is fixed. // Hadoop does not handle non-splittable files correctly for CombineFileInputFormat, // so don't use CombineFileInputFormat for non-splittable files - FileSystem inpFs = path.getFileSystem(job); + inpFs = path.getFileSystem(job); if (inputFormat instanceof TextInputFormat) { - Queue dirs = new LinkedList(); - FileStatus fStats = inpFs.getFileStatus(path); + dirs = new LinkedList(); + fStats = inpFs.getFileStatus(path); // If path is a directory if (fStats.isDir()) { @@ -326,8 +339,8 @@ else if ((new CompressionCodecFactory(job)).getCodec(path) != null) { } while (dirs.peek() != null) { - Path tstPath = dirs.remove(); - FileStatus[] fStatus = inpFs.listStatus(tstPath); + tstPath = dirs.remove(); + fStatus = inpFs.listStatus(tstPath); for (int idx = 0; idx < fStatus.length; idx++) { if (fStatus[idx].isDir()) { dirs.offer(fStatus[idx].getPath()); @@ -344,11 +357,11 @@ else if ((new CompressionCodecFactory(job)).getCodec(fStatus[idx].getPath()) != return super.getSplits(job, numSplits); } - Path filterPath = path; + filterPath = path; // Does a pool exist for this path already - CombineFilter f = null; - List> opList = null; + f = null; + opList = null; if (!mrwork.isMapperCannotSpanPartns()) { opList = HiveFileFormatUtils.doGetWorksFromPath( @@ -394,8 +407,8 @@ else if ((new CompressionCodecFactory(job)).getCodec(fStatus[idx].getPath()) != if (inpFiles.size() > 0) { // Processing files - for (Path filterPath : poolSet) { - combine.createPool(job, new CombineFilter(filterPath)); + for (Path fp : poolSet) { + combine.createPool(job, new CombineFilter(fp)); } processPaths(job, combine, iss, inpFiles.toArray(new Path[0])); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java index 1b99781..abbd776 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java @@ -314,11 +314,14 @@ private static boolean pathsContainNoScheme(Map pathToPar private static void populateNewPartitionDesc( Map pathToPartitionInfo, Map newPathToPartitionInfo) { + String entryKey, pathOnly; + PartitionDesc partDesc; + Path newP; for (Map.Entry entry: pathToPartitionInfo.entrySet()) { - String entryKey = entry.getKey(); - PartitionDesc partDesc = entry.getValue(); - Path newP = new Path(entryKey); - String pathOnly = newP.toUri().getPath(); + entryKey = entry.getKey(); + partDesc = entry.getValue(); + newP = new Path(entryKey); + pathOnly = newP.toUri().getPath(); newPathToPartitionInfo.put(pathOnly, partDesc); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java index b04b34e..6fc4f3c 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java @@ -268,19 +268,25 @@ protected void init(JobConf job) { ArrayList result = new ArrayList(); // for each dir, get the InputFormat, and do getSplits. + PartitionDesc part; + Class inputFormatClass; + InputFormat inputFormat; + ArrayList aliases; + Operator op; + InputSplit[] iss; for (Path dir : dirs) { - PartitionDesc part = getPartitionDescFromPath(pathToPartitionInfo, dir); + part = getPartitionDescFromPath(pathToPartitionInfo, dir); // create a new InputFormat instance if this is the first time to see this // class - Class inputFormatClass = part.getInputFileFormatClass(); - InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job); + inputFormatClass = part.getInputFileFormatClass(); + inputFormat = getInputFormatFromCache(inputFormatClass, job); Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), newjob); // Make filter pushdown information available to getSplits. - ArrayList aliases = + aliases = mrwork.getPathToAliases().get(dir.toUri().toString()); if ((aliases != null) && (aliases.size() == 1)) { - Operator op = mrwork.getAliasToWork().get(aliases.get(0)); + op = mrwork.getAliasToWork().get(aliases.get(0)); if ((op != null) && (op instanceof TableScanOperator)) { TableScanOperator tableScan = (TableScanOperator) op; pushFilters(newjob, tableScan); @@ -289,12 +295,11 @@ protected void init(JobConf job) { FileInputFormat.setInputPaths(newjob, dir); newjob.setInputFormat(inputFormat.getClass()); - InputSplit[] iss = inputFormat.getSplits(newjob, numSplits / dirs.length); + iss = inputFormat.getSplits(newjob, numSplits / dirs.length); for (InputSplit is : iss) { result.add(new HiveInputSplit(is, inputFormatClass.getName())); } } - LOG.info("number of splits " + result.size()); return result.toArray(new HiveInputSplit[result.size()]); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java index b7b48d4..7dd7c1d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java @@ -252,8 +252,9 @@ public int compare(HiveLockObj o1, HiveLockObj o2) { public void releaseLocks(List hiveLocks) { if (hiveLocks != null) { int len = hiveLocks.size(); + HiveLock hiveLock; for (int pos = len-1; pos >= 0; pos--) { - HiveLock hiveLock = hiveLocks.get(pos); + hiveLock = hiveLocks.get(pos); try { LOG.info(" about to release lock for " + hiveLock.getHiveLockObject().getName()); unlock(hiveLock); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 21f4fc2..c3af024 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -345,14 +345,16 @@ public void createTable(String tableName, List columns, tbl.setInputFormatClass(fileInputFormat.getName()); tbl.setOutputFormatClass(fileOutputFormat.getName()); + FieldSchema field; for (String col : columns) { - FieldSchema field = new FieldSchema(col, STRING_TYPE_NAME, "default"); + field = new FieldSchema(col, STRING_TYPE_NAME, "default"); tbl.getCols().add(field); } + FieldSchema part; if (partCols != null) { for (String partCol : partCols) { - FieldSchema part = new FieldSchema(); + part = new FieldSchema(); part.setName(partCol); part.setType(STRING_TYPE_NAME); // default partition key tbl.getPartCols().add(part); @@ -730,8 +732,9 @@ public void createIndex(String tableName, String indexName, String indexHandlerC storageDescriptor.setBucketCols(null); int k = 0; Table metaBaseTbl = new Table(baseTbl); + FieldSchema col; for (int i = 0; i < metaBaseTbl.getCols().size(); i++) { - FieldSchema col = metaBaseTbl.getCols().get(i); + col = metaBaseTbl.getCols().get(i); if (indexedCols.contains(col.getName())) { indexTblCols.add(col); sortCols.add(new Order(col.getName(), 1)); @@ -1388,14 +1391,16 @@ private void constructOneLBLocationMap(FileStatus fSta, // for each dynamically created DP directory, construct a full partition spec // and load the partition based on that Iterator iter = validPartitions.iterator(); + Path partPath; + LinkedHashMap fullPartSpec; while (iter.hasNext()) { // get the dynamically created directory - Path partPath = iter.next(); + partPath = iter.next(); assert fs.getFileStatus(partPath).isDir(): "partitions " + partPath + " is not a directory !"; // generate a full partition specification - LinkedHashMap fullPartSpec = new LinkedHashMap(partSpec); + fullPartSpec = new LinkedHashMap(partSpec); Warehouse.makeSpecFromName(fullPartSpec, partPath); fullPartSpecs.add(fullPartSpec); @@ -1819,8 +1824,9 @@ public boolean dropPartition(String db_name, String tbl_name, int nBatches = nParts / batchSize; try { + List tParts; for (int i = 0; i < nBatches; ++i) { - List tParts = + tParts = getMSC().getPartitionsByNames(tbl.getDbName(), tbl.getTableName(), partNames.subList(i*batchSize, (i+1)*batchSize)); if (tParts != null) { @@ -1831,7 +1837,7 @@ public boolean dropPartition(String db_name, String tbl_name, } if (nParts > nBatches * batchSize) { - List tParts = + tParts = getMSC().getPartitionsByNames(tbl.getDbName(), tbl.getTableName(), partNames.subList(nBatches*batchSize, nParts)); if (tParts != null) { @@ -2039,8 +2045,10 @@ public PrincipalPrivilegeSet get_privilege_set(HiveObjectType objectType, throw new HiveException("checkPaths: destination " + destf + " should be a directory"); } + FileStatus[] items; + List srcToDest; + Path itemSource; for (FileStatus src : srcs) { - FileStatus[] items; if (src.isDir()) { items = fs.listStatus(src.getPath()); Arrays.sort(items); @@ -2048,10 +2056,9 @@ public PrincipalPrivilegeSet get_privilege_set(HiveObjectType objectType, items = new FileStatus[] {src}; } - List srcToDest = new ArrayList(); + srcToDest = new ArrayList(); for (FileStatus item : items) { - - Path itemSource = item.getPath(); + itemSource = item.getPath(); if (Utilities.isTempPath(item)) { // This check is redundant because temp files are removed by diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index f1d29f8..cf79673 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -194,15 +194,17 @@ public void checkValidity() throws HiveException { Iterator iterCols = getCols().iterator(); List colNames = new ArrayList(); + String colName, oldColName; + Iterator iter; while (iterCols.hasNext()) { - String colName = iterCols.next().getName(); + colName = iterCols.next().getName(); if (!MetaStoreUtils.validateName(colName)) { throw new HiveException("Invalid column name '" + colName + "' in the table definition"); } - Iterator iter = colNames.iterator(); + iter = colNames.iterator(); while (iter.hasNext()) { - String oldColName = iter.next(); + oldColName = iter.next(); if (colName.equalsIgnoreCase(oldColName)) { throw new HiveException("Duplicate column name " + colName + " in the table definition."); @@ -213,9 +215,10 @@ public void checkValidity() throws HiveException { if (getPartCols() != null) { // there is no overlap between columns and partitioning columns + String partCol; Iterator partColsIter = getPartCols().iterator(); while (partColsIter.hasNext()) { - String partCol = partColsIter.next().getName(); + partCol = partColsIter.next().getName(); if (colNames.contains(partCol.toLowerCase())) { throw new HiveException("Partition column name " + partCol + " conflicts with table columns."); @@ -834,9 +837,11 @@ public boolean isIndexTable() { List fsl = getPartCols(); List tpl = tp.getValues(); LinkedHashMap spec = new LinkedHashMap(); + FieldSchema fs; + String value; for (int i = 0; i < fsl.size(); i++) { - FieldSchema fs = fsl.get(i); - String value = tpl.get(i); + fs = fsl.get(i); + value = tpl.get(i); spec.put(fs.getName(), value); } return spec; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java index 0f48674..464f946 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java @@ -180,13 +180,19 @@ public void showTableStatus(DataOutputStream outStream, { try { Iterator iterTables = tbls.iterator(); + String tableName, tblLoc, inputFormattCls, outputFormattCls, owner, ddlCols, partitionCols; + Table tbl; + List cols; + boolean isPartitioned; + List locations; + Path tblPath; while (iterTables.hasNext()) { // create a row per table name - Table tbl = iterTables.next(); - String tableName = tbl.getTableName(); - String tblLoc = null; - String inputFormattCls = null; - String outputFormattCls = null; + tbl = iterTables.next(); + tableName = tbl.getTableName(); + tblLoc = null; + inputFormattCls = null; + outputFormattCls = null; if (part != null) { if (par != null) { if (par.getLocation() != null) { @@ -203,11 +209,11 @@ public void showTableStatus(DataOutputStream outStream, outputFormattCls = tbl.getOutputFormatClass().getName(); } - String owner = tbl.getOwner(); - List cols = tbl.getCols(); - String ddlCols = MetaStoreUtils.getDDLFromFieldSchema("columns", cols); - boolean isPartitioned = tbl.isPartitioned(); - String partitionCols = ""; + owner = tbl.getOwner(); + cols = tbl.getCols(); + ddlCols = MetaStoreUtils.getDDLFromFieldSchema("columns", cols); + isPartitioned = tbl.isPartitioned(); + partitionCols = ""; if (isPartitioned) { partitionCols = MetaStoreUtils.getDDLFromFieldSchema( "partition_columns", tbl.getPartCols()); @@ -230,8 +236,8 @@ public void showTableStatus(DataOutputStream outStream, outStream.writeBytes("partitionColumns:" + partitionCols); outStream.write(terminator); // output file system information - Path tblPath = tbl.getPath(); - List locations = new ArrayList(); + tblPath = tbl.getPath(); + locations = new ArrayList(); if (isPartitioned) { if (par == null) { for (Partition curPart : db.getPartitions(tbl)) { @@ -291,12 +297,15 @@ private void writeFileSystemStats(DataOutputStream outStream, } if (!unknown) { + FileStatus status; + FileStatus[] files; + long accessTime, updateTime, fileLen; for (Path loc : locations) { try { - FileStatus status = fs.getFileStatus(tblPath); - FileStatus[] files = fs.listStatus(loc); - long accessTime = ShimLoader.getHadoopShims().getAccessTime(status); - long updateTime = status.getModificationTime(); + status = fs.getFileStatus(tblPath); + files = fs.listStatus(loc); + accessTime = ShimLoader.getHadoopShims().getAccessTime(status); + updateTime = status.getModificationTime(); // no matter loc is the table location or part location, it must be a // directory. if (!status.isDir()) { @@ -313,7 +322,7 @@ private void writeFileSystemStats(DataOutputStream outStream, continue; } numOfFiles++; - long fileLen = currentStatus.getLen(); + fileLen = currentStatus.getLen(); totalFileSize += fileLen; if (fileLen > maxFileSize) { maxFileSize = fileLen; @@ -394,10 +403,11 @@ public void showTablePartitons(DataOutputStream outStream, List parts) throws HiveException { try { + SessionState ss; for (String part : parts) { // Partition names are URL encoded. We decode the names unless Hive // is configured to use the encoded names. - SessionState ss = SessionState.get(); + ss = SessionState.get(); if (ss != null && ss.getConf() != null && !ss.getConf().getBoolVar(HiveConf.ConfVars.HIVE_DECODE_PARTITION_NAME)) { outStream.writeBytes(part); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java index 0991847..0a13089 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java @@ -235,9 +235,10 @@ protected boolean checkConvertBucketMapJoin( // For nested sub-queries, the alias mapping is not maintained in QB currently. if (topOps.containsValue(tso)) { + String newAlias; for (Map.Entry> topOpEntry : topOps.entrySet()) { if (topOpEntry.getValue() == tso) { - String newAlias = topOpEntry.getKey(); + newAlias = topOpEntry.getKey(); joinAliases.set(index, newAlias); if (baseBigAlias.equals(alias)) { baseBigAlias = newAlias; @@ -401,18 +402,25 @@ protected void convertMapJoinToBucketMapJoin( // go through all small tables and get the mapping from bucket file name // in the big table to bucket file names in small tables. + String alias; + List smallTblBucketNums; + List> smallTblFilesList; + Map> mappingBigTableBucketFileNameToSmallTableBucketFileNames; + Iterator> bigTblPartToBucketNum; + Iterator>> bigTblPartToBucketNames; + List bigTblBucketNameList; for (int j = 0; j < joinAliases.size(); j++) { - String alias = joinAliases.get(j); + alias = joinAliases.get(j); if (alias.equals(baseBigAlias)) { continue; } for (List names : tblAliasToBucketedFilePathsInEachPartition.get(alias)) { Collections.sort(names); } - List smallTblBucketNums = tblAliasToNumberOfBucketsInEachPartition.get(alias); - List> smallTblFilesList = tblAliasToBucketedFilePathsInEachPartition.get(alias); + smallTblBucketNums = tblAliasToNumberOfBucketsInEachPartition.get(alias); + smallTblFilesList = tblAliasToBucketedFilePathsInEachPartition.get(alias); - Map> mappingBigTableBucketFileNameToSmallTableBucketFileNames = + mappingBigTableBucketFileNameToSmallTableBucketFileNames = new LinkedHashMap>(); aliasBucketFileNameMapping.put(alias, mappingBigTableBucketFileNameToSmallTableBucketFileNames); @@ -420,14 +428,14 @@ protected void convertMapJoinToBucketMapJoin( // for each bucket file in big table, get the corresponding bucket file // name in the small table. // more than 1 partition in the big table, do the mapping for each partition - Iterator>> bigTblPartToBucketNames = + bigTblPartToBucketNames = bigTblPartsToBucketFileNames.entrySet().iterator(); - Iterator> bigTblPartToBucketNum = bigTblPartsToBucketNumber + bigTblPartToBucketNum = bigTblPartsToBucketNumber .entrySet().iterator(); while (bigTblPartToBucketNames.hasNext()) { assert bigTblPartToBucketNum.hasNext(); int bigTblBucketNum = bigTblPartToBucketNum.next().getValue(); - List bigTblBucketNameList = bigTblPartToBucketNames.next().getValue(); + bigTblBucketNameList = bigTblPartToBucketNames.next().getValue(); fillMappingBigTableBucketFileNameToSmallTableBucketFileNames(smallTblBucketNums, smallTblFilesList, mappingBigTableBucketFileNameToSmallTableBucketFileNames, bigTblBucketNum, @@ -474,11 +482,14 @@ private static void fillMappingBigTableBucketFileNameToSmallTableBucketFileNames int bigTblBucketNum, List bigTblBucketNameList, Map bucketFileNameMapping) { + ArrayList resultFileNames; + List smallTblFileNames; + int smallTblBucketNum, jump; for (int bindex = 0; bindex < bigTblBucketNameList.size(); bindex++) { - ArrayList resultFileNames = new ArrayList(); + resultFileNames = new ArrayList(); for (int sindex = 0; sindex < smallTblBucketNums.size(); sindex++) { - int smallTblBucketNum = smallTblBucketNums.get(sindex); - List smallTblFileNames = smallTblFilesList.get(sindex); + smallTblBucketNum = smallTblBucketNums.get(sindex); + smallTblFileNames = smallTblFilesList.get(sindex); if (bigTblBucketNum >= smallTblBucketNum) { // if the big table has more buckets than the current small table, // use "MOD" to get small table bucket names. For example, if the big @@ -487,7 +498,7 @@ private static void fillMappingBigTableBucketFileNameToSmallTableBucketFileNames int toAddSmallIndex = bindex % smallTblBucketNum; resultFileNames.add(smallTblFileNames.get(toAddSmallIndex)); } else { - int jump = smallTblBucketNum / bigTblBucketNum; + jump = smallTblBucketNum / bigTblBucketNum; for (int i = bindex; i < smallTblFileNames.size(); i = i + jump) { resultFileNames.add(smallTblFileNames.get(i)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java index fda2f84..3e0cb4c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java @@ -178,23 +178,28 @@ protected SMBMapJoinOperator convertBucketMapJoinToSMBJoin(MapJoinOperator mapJo */ List> parentOperators = mapJoinOp.getParentOperators(); + Operator par; + int index; + DummyStoreOperator dummyStoreOp; + List> childrenOps; + List> parentOps; for (int i = 0; i < parentOperators.size(); i++) { - Operator par = parentOperators.get(i); - int index = par.getChildOperators().indexOf(mapJoinOp); + par = parentOperators.get(i); + index = par.getChildOperators().indexOf(mapJoinOp); par.getChildOperators().remove(index); if (i == smbJoinDesc.getPosBigTable()) { par.getChildOperators().add(index, smbJop); } else { - DummyStoreOperator dummyStoreOp = new DummyStoreOperator(); + dummyStoreOp = new DummyStoreOperator(); par.getChildOperators().add(index, dummyStoreOp); - List> childrenOps = + childrenOps = new ArrayList>(); childrenOps.add(smbJop); dummyStoreOp.setChildOperators(childrenOps); - List> parentOps = + parentOps = new ArrayList>(); parentOps.add(par); dummyStoreOp.setParentOperators(parentOps); @@ -207,9 +212,10 @@ protected SMBMapJoinOperator convertBucketMapJoinToSMBJoin(MapJoinOperator mapJo smbJoinDesc.setAliasToSink(aliasToSink); List> childOps = mapJoinOp.getChildOperators(); + Operator child; for (int i = 0; i < childOps.size(); i++) { - Operator child = childOps.get(i); - int index = child.getParentOperators().indexOf(mapJoinOp); + child = childOps.get(i); + index = child.getParentOperators().indexOf(mapJoinOp); child.getParentOperators().remove(index); child.getParentOperators().add(index, smbJop); } @@ -478,10 +484,13 @@ protected boolean canConvertJoinToBucketMapJoin( Map> keyExprMap = new HashMap>(); List> parentOps = joinOp.getParentOperators(); // get the join keys from parent ReduceSink operators + ReduceSinkDesc rsconf; + Byte tag; + List keys; for (Operator parentOp : parentOps) { - ReduceSinkDesc rsconf = ((ReduceSinkOperator)parentOp).getConf(); - Byte tag = (byte) rsconf.getTag(); - List keys = rsconf.getKeyCols(); + rsconf = ((ReduceSinkOperator)parentOp).getConf(); + tag = (byte) rsconf.getTag(); + keys = rsconf.getKeyCols(); keyExprMap.put(tag, keys); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index 4bbaf2b..4979a96 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -534,11 +534,14 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, if (rr.getColumnInfos().size() != cols.size()) { ArrayList colList = new ArrayList(); ArrayList outputColNames = new ArrayList(); + String[] tabcol; + ColumnInfo colInfo; + ExprNodeColumnDesc colExpr; for (String col : cols) { // revert output cols of SEL(*) to ExprNodeColumnDesc - String[] tabcol = rr.reverseLookup(col); - ColumnInfo colInfo = rr.get(tabcol[0], tabcol[1]); - ExprNodeColumnDesc colExpr = new ExprNodeColumnDesc(colInfo.getType(), + tabcol = rr.reverseLookup(col); + colInfo = rr.get(tabcol[0], tabcol[1]); + colExpr = new ExprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName(), colInfo.getTabAlias(), colInfo.getIsVirtualCol()); colList.add(colExpr); outputColNames.add(col); @@ -858,11 +861,14 @@ private static void pruneJoinOperator(NodeProcessorCtx ctx, Set>> filters = conf.getFilters().entrySet(); Iterator>> iter = filters.iterator(); + List cols; + Byte tag; + Map.Entry> entry; while (iter.hasNext()) { - Map.Entry> entry = iter.next(); - Byte tag = entry.getKey(); + entry = iter.next(); + tag = entry.getKey(); for (ExprNodeDesc desc : entry.getValue()) { - List cols = prunedColLists.get(tag); + cols = prunedColLists.get(tag); cols = Utilities.mergeUniqElems(cols, desc.getCols()); prunedColLists.put(tag, cols); } @@ -874,10 +880,13 @@ private static void pruneJoinOperator(NodeProcessorCtx ctx, ArrayList rs = new ArrayList(); Map newColExprMap = new HashMap(); + String internalName; + ExprNodeDesc desc; + List prunedRSList; for (int i = 0; i < conf.getOutputColumnNames().size(); i++) { - String internalName = conf.getOutputColumnNames().get(i); - ExprNodeDesc desc = columnExprMap.get(internalName); - Byte tag = conf.getReversedExprs().get(internalName); + internalName = conf.getOutputColumnNames().get(i); + desc = columnExprMap.get(internalName); + tag = conf.getReversedExprs().get(internalName); if (!childColLists.contains(internalName)) { int index = conf.getExprs().get(tag).indexOf(desc); if (index < 0) { @@ -888,7 +897,7 @@ private static void pruneJoinOperator(NodeProcessorCtx ctx, retainMap.get(tag).remove(index); } } else { - List prunedRSList = prunedColLists.get(tag); + prunedRSList = prunedColLists.get(tag); if (prunedRSList == null) { prunedRSList = new ArrayList(); prunedColLists.put(tag, prunedRSList); @@ -902,31 +911,37 @@ private static void pruneJoinOperator(NodeProcessorCtx ctx, if (mapJoin) { // regenerate the valueTableDesc List valueTableDescs = new ArrayList(); + List valueCols; + StringBuilder keyOrder; + TableDesc valueTableDesc; + Set>> exprs; + Iterator>> iters; + List lists; for (int pos = 0; pos < op.getParentOperators().size(); pos++) { - List valueCols = conf.getExprs() + valueCols = conf.getExprs() .get(Byte.valueOf((byte) pos)); - StringBuilder keyOrder = new StringBuilder(); + keyOrder = new StringBuilder(); for (int i = 0; i < valueCols.size(); i++) { keyOrder.append("+"); } - TableDesc valueTableDesc = PlanUtils.getMapJoinValueTableDesc(PlanUtils + valueTableDesc = PlanUtils.getMapJoinValueTableDesc(PlanUtils .getFieldSchemasFromColumnList(valueCols, "mapjoinvalue")); valueTableDescs.add(valueTableDesc); } ((MapJoinDesc) conf).setValueTblDescs(valueTableDescs); - Set>> exprs = ((MapJoinDesc) conf) + exprs = ((MapJoinDesc) conf) .getKeys().entrySet(); - Iterator>> iters = exprs.iterator(); + iters = exprs.iterator(); while (iters.hasNext()) { - Map.Entry> entry = iters.next(); - List lists = entry.getValue(); + entry = iters.next(); + lists = entry.getValue(); for (int j = 0; j < lists.size(); j++) { - ExprNodeDesc desc = lists.get(j); - Byte tag = entry.getKey(); - List cols = prunedColLists.get(tag); + desc = lists.get(j); + tag = entry.getKey(); + cols = prunedColLists.get(tag); cols = Utilities.mergeUniqElems(cols, desc.getCols()); prunedColLists.put(tag, cols); } @@ -934,18 +949,21 @@ private static void pruneJoinOperator(NodeProcessorCtx ctx, } + boolean[] flags; for (Operator child : childOperators) { if (child instanceof ReduceSinkOperator) { - boolean[] flags = getPruneReduceSinkOpRetainFlags(childColLists, + flags = getPruneReduceSinkOpRetainFlags(childColLists, (ReduceSinkOperator) child); pruneReduceSinkOperator(flags, (ReduceSinkOperator) child, cppCtx); } } + String[] nm; + ColumnInfo col; for (int i = 0; i < outputCols.size(); i++) { - String internalName = outputCols.get(i); - String[] nm = joinRR.reverseLookup(internalName); - ColumnInfo col = joinRR.get(nm[0], nm[1]); + internalName = outputCols.get(i); + nm = joinRR.reverseLookup(internalName); + col = joinRR.get(nm[0], nm[1]); newJoinRR.put(nm[0], nm[1], col); rs.add(col); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java index d35696d..6334fcc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java @@ -65,11 +65,12 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { parts = partsList.getConfirmedPartns(); parts.addAll(partsList.getUnknownPartns()); if ((parts != null) && (parts.size() > 0)) { + NodeProcessorCtx opWalkerCtx; for (Partition part : parts) { // only process partition which is skewed and list bucketed if (ListBucketingPrunerUtils.isListBucketingPart(part)) { // create a the context for walking operators - NodeProcessorCtx opWalkerCtx = new LBOpWalkerCtx(pctx.getOpToPartToSkewedPruner(), + opWalkerCtx = new LBOpWalkerCtx(pctx.getOpToPartToSkewedPruner(), part); // walk operator tree to create expression tree for list bucketing @@ -298,12 +299,13 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { List> skewedValues = part.getSkewedColValues(); List nonSkewedValueMatchResult = new ArrayList(); SkewedValueList skewedValueList = new SkewedValueList(); + Boolean matchResult; for (List cell : collections) { // Walk through the tree to decide value. // Example: skewed column: C1, C2 ; // index: (1,a) ; // expression tree: ((c1=1) and (c2=a)) or ((c1=3) or (c2=b)) - Boolean matchResult = ListBucketingPrunerUtils.evaluateExprOnCell(skewedCols, cell, pruner, + matchResult = ListBucketingPrunerUtils.evaluateExprOnCell(skewedCols, cell, pruner, uniqSkewedValues); // Handle skewed value. if (skewedValues.contains(cell)) { // if it is skewed value @@ -619,8 +621,9 @@ private static void walker(List> finalResult, final List oneCompleteIndex; for (String v : input.get(level)) { - List oneCompleteIndex = new ArrayList(listSoFar); + oneCompleteIndex = new ArrayList(listSoFar); oneCompleteIndex.add(v); finalResult.add(oneCompleteIndex); } @@ -628,8 +631,9 @@ private static void walker(List> finalResult, final List clonedListSoFar; for (String v : input.get(level)) { - List clonedListSoFar = new ArrayList(listSoFar); + clonedListSoFar = new ArrayList(listSoFar); clonedListSoFar.add(v); int nextLevel = level + 1; walker(finalResult, input, clonedListSoFar, nextLevel); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AbstractJoinTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AbstractJoinTaskDispatcher.java index 33ef581..24f2dea 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AbstractJoinTaskDispatcher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/AbstractJoinTaskDispatcher.java @@ -132,15 +132,20 @@ public long getTotalKnownInputSize(Context context, MapWork currWork, // is chosen as big table, what's the total size of left tables, which // are going to be small tables. long aliasTotalKnownInputSize = 0L; + String path; + List aliasList; + ContentSummary contentSummary; + long size; + Long es; for (Map.Entry> entry : pathToAliases.entrySet()) { - String path = entry.getKey(); - List aliasList = entry.getValue(); - ContentSummary cs = context.getCS(path); - if (cs != null) { - long size = cs.getLength(); + path = entry.getKey(); + aliasList = entry.getValue(); + contentSummary = context.getCS(path); + if (contentSummary != null) { + size = contentSummary.getLength(); for (String alias : aliasList) { aliasTotalKnownInputSize += size; - Long es = aliasToSize.get(alias); + es = aliasToSize.get(alias); if (es == null) { es = new Long(0); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java index 87fba2d..c025d7a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java @@ -77,6 +77,13 @@ public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException { * @throws SemanticException */ private void inferBucketingSorting(List mapRedTasks) throws SemanticException { + Operator reducer; + BucketingSortingCtx bCtx; + boolean disableBucketing; + Map opRules; + Dispatcher disp; + GraphWalker ogw; + ArrayList topNodes; for (ExecDriver mapRedTask : mapRedTasks) { // For now this only is used to determine the bucketing/sorting of outputs, in the future @@ -89,18 +96,18 @@ private void inferBucketingSorting(List mapRedTasks) throws Semantic if (mapRedTask.getWork().getReduceWork() == null) { continue; } - Operator reducer = mapRedTask.getWork().getReduceWork().getReducer(); + reducer = mapRedTask.getWork().getReduceWork().getReducer(); // uses sampling, which means it's not bucketed - boolean disableBucketing = mapRedTask.getWork().getMapWork().getSamplingType() > 0; - BucketingSortingCtx bCtx = new BucketingSortingCtx(disableBucketing); + disableBucketing = mapRedTask.getWork().getMapWork().getSamplingType() > 0; + bCtx = new BucketingSortingCtx(disableBucketing); // RuleRegExp rules are used to match operators anywhere in the tree // RuleExactMatch rules are used to specify exactly what the tree should look like // In particular, this guarantees that the first operator is the reducer // (and its parent(s) are ReduceSinkOperators) since it begins walking the tree from // the reducer. - Map opRules = new LinkedHashMap(); + opRules = new LinkedHashMap(); opRules.put(new RuleRegExp("R1", SelectOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getSelProc()); // Matches only GroupByOpeartors which are reducers, rather than map group by operators, @@ -136,12 +143,12 @@ private void inferBucketingSorting(List mapRedTasks) throws Semantic // The dispatcher fires the processor corresponding to the closest matching rule and passes // the context along - Dispatcher disp = new DefaultRuleDispatcher(BucketingSortingOpProcFactory.getDefaultProc(), + disp = new DefaultRuleDispatcher(BucketingSortingOpProcFactory.getDefaultProc(), opRules, bCtx); - GraphWalker ogw = new PreOrderWalker(disp); + ogw = new PreOrderWalker(disp); // Create a list of topop nodes - ArrayList topNodes = new ArrayList(); + topNodes = new ArrayList(); topNodes.add(reducer); ogw.startWalking(topNodes, null); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java index e0f128b..0281427 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java @@ -148,6 +148,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, List bucketCols = new ArrayList(); List sortCols = new ArrayList(); // Go through the Reduce keys and find the matching column(s) in the reduce values + String colName; for (int keyIndex = 0; keyIndex < rop.getConf().getKeyCols().size(); keyIndex++) { for (int valueIndex = 0; valueIndex < rop.getConf().getValueCols().size(); valueIndex++) { @@ -156,7 +157,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, equals(new ExprNodeDescEqualityWrapper(rop.getConf().getKeyCols().get( keyIndex)))) { - String colName = rop.getSchema().getSignature().get(valueIndex).getInternalName(); + colName = rop.getSchema().getSignature().get(valueIndex).getInternalName(); bucketCols.add(new BucketCol(colName, keyIndex)); sortCols.add(new SortCol(colName, keyIndex, sortOrder.charAt(keyIndex))); break; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java index e214807..d516067 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java @@ -495,6 +495,11 @@ public static boolean cannotConvert(String bigTableAlias, long ThresholdOfSmallTblSizeSum = HiveConf.getLongVar(conf, HiveConf.ConfVars.HIVESMALLTABLESFILESIZE); + String xml; + InputStream in; + MapredWork newWork; + ObjectPair newTaskAlias; + MapRedTask newTask; for (int i = 0; i < numAliases; i++) { // this table cannot be big table if (!bigTableCandidates.contains(i)) { @@ -502,13 +507,13 @@ public static boolean cannotConvert(String bigTableAlias, } // deep copy a new mapred work from xml // Once HIVE-4396 is in, it would be faster to use a cheaper method to clone the plan - String xml = currTask.getWork().toXML(); - InputStream in = new ByteArrayInputStream(xml.getBytes("UTF-8")); - MapredWork newWork = Utilities.deserializeObject(in); + xml = currTask.getWork().toXML(); + in = new ByteArrayInputStream(xml.getBytes("UTF-8")); + newWork = Utilities.deserializeObject(in); // create map join task and set big table as i - ObjectPair newTaskAlias = convertTaskToMapJoinTask(newWork, i); - MapRedTask newTask = newTaskAlias.getFirst(); + newTaskAlias = convertTaskToMapJoinTask(newWork, i); + newTask = newTaskAlias.getFirst(); bigTableAlias = newTaskAlias.getSecond(); if (cannotConvert(bigTableAlias, aliasToSize, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java index 85a2bdf..64f0630 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java @@ -143,21 +143,25 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, Object.. new ArrayList>(); // get all parents List> parentsOp = mapJoinOp.getParentOperators(); + Operator parent; + HashTableDummyDesc desc; + HashTableDummyOperator dummyOp; + TableDesc tbl; + List> dummyChildren; for (int i = 0; i < parentsOp.size(); i++) { if (i == bigTable) { smallTablesParentOp.add(null); continue; } - Operator parent = parentsOp.get(i); + parent = parentsOp.get(i); // let hashtable Op be the child of this parent parent.replaceChild(mapJoinOp, hashTableSinkOp); // keep the parent id correct smallTablesParentOp.add(parent); // create an new operator: HashTable DummyOpeator, which share the table desc - HashTableDummyDesc desc = new HashTableDummyDesc(); - HashTableDummyOperator dummyOp = (HashTableDummyOperator) OperatorFactory.get(desc); - TableDesc tbl; + desc = new HashTableDummyDesc(); + dummyOp = (HashTableDummyOperator) OperatorFactory.get(desc); if (parent.getSchema() == null) { if (parent instanceof TableScanOperator) { @@ -174,7 +178,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, Object.. dummyOp.getConf().setTbl(tbl); // let the dummy op be the parent of mapjoin op mapJoinOp.replaceParent(parent, dummyOp); - List> dummyChildren = + dummyChildren = new ArrayList>(); dummyChildren.add(mapJoinOp); dummyOp.setChildOperators(dummyChildren); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java index 5c6751c..319d904 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java @@ -125,9 +125,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, List tableIndexes = indexes.get(srcTable); Map> indexesByType = new HashMap>(); + List newType; for (Index indexOnTable : tableIndexes) { if (indexesByType.get(indexOnTable.getIndexHandlerClass()) == null) { - List newType = new ArrayList(); + newType = new ArrayList(); newType.add(indexOnTable); indexesByType.put(indexOnTable.getIndexHandlerClass(), newType); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 70975c6..8c0f00d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -125,8 +125,9 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { partColNames.add(fsc.getName()); } List partitions = rv.getValue(); + AddPartitionDesc partDesc; for (Partition partition : partitions) { - AddPartitionDesc partDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(), + partDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); partDesc.setInputFormat(partition.getSd().getInputFormat()); @@ -166,10 +167,12 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { if (child.getChildCount() == 2) { ASTNode partspec = (ASTNode) child.getChild(1); // partSpec is a mapping from partition column name to its value. + ASTNode partspec_val; + String val, colName; for (int j = 0; j < partspec.getChildCount(); ++j) { - ASTNode partspec_val = (ASTNode) partspec.getChild(j); - String val = null; - String colName = unescapeIdentifier(partspec_val.getChild(0) + partspec_val = (ASTNode) partspec.getChild(j); + val = null; + colName = unescapeIdentifier(partspec_val.getChild(0) .getText().toLowerCase()); if (partspec_val.getChildCount() < 2) { // DP in the form of T // partition (ds, hr) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java index a8d1a68..e021efb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java @@ -92,11 +92,15 @@ public IndexUpdater(LoadTableDesc loadTableWork, Set inputs, public List> generateUpdateTasks() throws HiveException { hive = Hive.get(this.conf); + TableDesc td; + Table srcTable; + List tblIndexes; + Map partSpec; for (LoadTableDesc ltd : loadTableWork) { - TableDesc td = ltd.getTable(); - Table srcTable = hive.getTable(td.getTableName()); - List tblIndexes = srcTable.getAllIndexes((short)-1); - Map partSpec = ltd.getPartitionSpec(); + td = ltd.getTable(); + srcTable = hive.getTable(td.getTableName()); + tblIndexes = srcTable.getAllIndexes((short)-1); + partSpec = ltd.getPartitionSpec(); if (partSpec == null || partSpec.size() == 0) { //unpartitioned table, update whole index doIndexUpdate(tblIndexes); @@ -109,8 +113,9 @@ public IndexUpdater(LoadTableDesc loadTableWork, Set inputs, private void doIndexUpdate(List tblIndexes) throws HiveException { Driver driver = new Driver(this.conf); + StringBuilder sb; for (Index idx : tblIndexes) { - StringBuilder sb = new StringBuilder(); + sb = new StringBuilder(); sb.append("ALTER INDEX "); sb.append(idx.getIndexName()); sb.append(" ON "); @@ -136,6 +141,8 @@ private void doIndexUpdate(Index index, Map partSpec) throws StringBuilder ps = new StringBuilder(); boolean first = true; ps.append("("); + StringBuilder sb; + Driver driver; for (String key : partSpec.keySet()) { if (!first) { ps.append(", "); @@ -147,7 +154,7 @@ private void doIndexUpdate(Index index, Map partSpec) throws ps.append(partSpec.get(key)); } ps.append(")"); - StringBuilder sb = new StringBuilder(); + sb = new StringBuilder(); sb.append("ALTER INDEX "); sb.append(index.getIndexName()); sb.append(" ON "); @@ -155,7 +162,7 @@ private void doIndexUpdate(Index index, Map partSpec) throws sb.append(" PARTITION "); sb.append(ps.toString()); sb.append(" REBUILD"); - Driver driver = new Driver(this.conf); + driver = new Driver(this.conf); driver.compile(sb.toString(), false); tasks.addAll(driver.getPlan().getRootTasks()); inputs.addAll(driver.getPlan().getInputs()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java index b42a425..f8f7727 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java @@ -100,8 +100,9 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) }); walker.startWalking(Collections.singletonList(expression), null); } + TypeInfo colType; for (FieldSchema argument : arguments) { - TypeInfo colType = + colType = TypeInfoUtils.getTypeInfoFromTypeString(argument.getType()); rowResolver.put("", argument.getName(), new ColumnInfo(argument.getName(), colType, "", false)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java index e3cab05..94a589a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java @@ -72,20 +72,26 @@ public void deriveBucketMapJoinMapping() { if (aliasBucketFileNameMapping != null) { aliasBucketBaseFileNameMapping = new LinkedHashMap>>(); + String tableAlias; + Map> baseFileNameMapping; + Map> fullPathMappings; + String inputPath; + List bucketPaths; + List bucketBaseFileNames; for (Map.Entry>> aliasToMappins : aliasBucketFileNameMapping.entrySet()) { - String tableAlias = aliasToMappins.getKey(); - Map> fullPathMappings = aliasToMappins.getValue(); + tableAlias = aliasToMappins.getKey(); + fullPathMappings = aliasToMappins.getValue(); - Map> baseFileNameMapping = new LinkedHashMap>(); + baseFileNameMapping = new LinkedHashMap>(); for (Map.Entry> inputToBuckets : fullPathMappings.entrySet()) { // For a given table and its bucket full file path list, // only keep the base file name (remove file path etc). // And put the new list into the new mapping. - String inputPath = inputToBuckets.getKey(); - List bucketPaths = inputToBuckets.getValue(); + inputPath = inputToBuckets.getKey(); + bucketPaths = inputToBuckets.getValue(); - List bucketBaseFileNames = new ArrayList(bucketPaths.size()); + bucketBaseFileNames = new ArrayList(bucketPaths.size()); //for each bucket file, only keep its base files and store into a new list. for (String bucketFName : bucketPaths) { bucketBaseFileNames.add(getBaseFileName(bucketFName)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java index 14fced7..72ed326 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java @@ -120,9 +120,11 @@ private void initRetainExprList() { retainList = new HashMap>(); Set>> set = exprs.entrySet(); Iterator>> setIter = set.iterator(); + Entry> current; + List list; while (setIter.hasNext()) { - Entry> current = setIter.next(); - List list = new ArrayList(); + current = setIter.next(); + list = new ArrayList(); for (int i = 0; i < current.getValue().size(); i++) { list.add(i); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java index dad5674..7c11739 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java @@ -209,9 +209,11 @@ public void setReversedExprs(Map reversedExprs) { LinkedHashMap ret = new LinkedHashMap(); + StringBuilder sb; + boolean first; for (Map.Entry> ent : getExprs().entrySet()) { - StringBuilder sb = new StringBuilder(); - boolean first = true; + sb = new StringBuilder(); + first = true; if (ent.getValue() != null) { for (ExprNodeDesc expr : ent.getValue()) { if (!first) { @@ -250,9 +252,11 @@ public void setExprs(final Map> exprs) { LinkedHashMap ret = new LinkedHashMap(); boolean filtersPresent = false; + StringBuilder sb; + boolean first; for (Map.Entry> ent : getFilters().entrySet()) { - StringBuilder sb = new StringBuilder(); - boolean first = true; + sb = new StringBuilder(); + first = true; if (ent.getValue() != null) { if (ent.getValue().size() != 0) { filtersPresent = true; @@ -490,9 +494,10 @@ public void setFilterMap(int[][] filterMap) { if (filterMap == null) { return null; } + boolean noFilter; for (int i = 0; i < filterMap.length; i++) { if (filterMap[i] != null) { - boolean noFilter = true; + noFilter = true; // join positions for even index, filter lengths for odd index for (int j = 1; j < filterMap[i].length; j += 2) { if (filterMap[i][j] > 0) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java index 31fcaeb..61f298c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ListBucketingCtx.java @@ -104,12 +104,13 @@ public void processRowSkewedIndex(RowSchema rowSch) { if ((this.skewedColNames != null) && (this.skewedColNames.size() > 0) && (rowSch != null) && (rowSch.getSignature() != null) && (rowSch.getSignature().size() > 0)) { List cols = rowSch.getSignature(); - int hitNo = 0; + int index, hitNo = 0; + SkewedColumnPositionPair pair; for (int i = 0; i < cols.size(); i++) { - int index = this.skewedColNames.indexOf(cols.get(i).getInternalName()); + index = this.skewedColNames.indexOf(cols.get(i).getInternalName()); if (index > -1) { hitNo++; - SkewedColumnPositionPair pair = new SkewedColumnPositionPair(i, index); + pair = new SkewedColumnPositionPair(i, index); rowSkewedIndex.add(pair); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java index e609633..014867d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java @@ -96,9 +96,11 @@ private void initRetainExprList() { retainList = new HashMap>(); Set>> set = super.getExprs().entrySet(); Iterator>> setIter = set.iterator(); + Entry> current; + List list; while (setIter.hasNext()) { - Entry> current = setIter.next(); - List list = new ArrayList(); + current = setIter.next(); + list = new ArrayList(); for (int i = 0; i < current.getValue().size(); i++) { list.add(i); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java index 58ea3ba..2bb84b6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java @@ -112,8 +112,9 @@ protected EWAHCompressedBitmap wordArrayToBitmap(Object b) { ListObjectInspector lloi = (ListObjectInspector) b1OI; int length = lloi.getListLength(b); ArrayList bitmapArray = new ArrayList(); + long l; for (int i = 0; i < length; i++) { - long l = PrimitiveObjectInspectorUtils.getLong( + l = PrimitiveObjectInspectorUtils.getLong( lloi.getListElement(b, i), (PrimitiveObjectInspector) lloi.getListElementObjectInspector()); bitmapArray.add(new LongWritable(l)); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/CheckColumnAccessHook.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/CheckColumnAccessHook.java index 14fc430..39643ea 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/CheckColumnAccessHook.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/CheckColumnAccessHook.java @@ -65,11 +65,13 @@ public void run(HookContext hookContext) { // This makes tests that use this hook deterministic. Map outputOrderedMap = new HashMap(); + StringBuilder perTableInfo; + String[] columns; for (Map.Entry> tableAccess : tableToColumnAccessMap.entrySet()) { - StringBuilder perTableInfo = new StringBuilder(); + perTableInfo = new StringBuilder(); perTableInfo.append("Table:").append(tableAccess.getKey()).append("\n"); // Sort columns to make output deterministic - String[] columns = new String[tableAccess.getValue().size()]; + columns = new String[tableAccess.getValue().size()]; tableAccess.getValue().toArray(columns); Arrays.sort(columns); perTableInfo.append("Columns:").append(StringUtils.join(columns, ',')) diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/CheckTableAccessHook.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/CheckTableAccessHook.java index 8e19fad..a66cf87 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/CheckTableAccessHook.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/CheckTableAccessHook.java @@ -65,9 +65,10 @@ public void run(HookContext hookContext) { // This makes tests that use this hook deterministic. Map outputOrderedMap = new HashMap(); + StringBuilder perOperatorInfo; for (Map.Entry, Map>> tableAccess: operatorToTableAccessMap.entrySet()) { - StringBuilder perOperatorInfo = new StringBuilder(); + perOperatorInfo = new StringBuilder(); perOperatorInfo.append("Operator:").append(tableAccess.getKey().getOperatorId()) .append("\n"); for (Map.Entry> entry: tableAccess.getValue().entrySet()) { @@ -78,8 +79,8 @@ public void run(HookContext hookContext) { outputOrderedMap.put(tableAccess.getKey().getOperatorId(), perOperatorInfo.toString()); } - for (String perOperatorInfo: outputOrderedMap.values()) { - console.printError(perOperatorInfo); + for (String pOinfo: outputOrderedMap.values()) { + console.printError(pOinfo); } } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyTableDirectoryIsEmptyHook.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyTableDirectoryIsEmptyHook.java index c2005e4..b229f36 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyTableDirectoryIsEmptyHook.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyTableDirectoryIsEmptyHook.java @@ -27,9 +27,11 @@ public class VerifyTableDirectoryIsEmptyHook implements ExecuteWithHookContext { public void run(HookContext hookContext) throws IOException { + Path tableLocation; + FileSystem fs; for (WriteEntity output : hookContext.getOutputs()) { - Path tableLocation = new Path(output.getTable().getDataLocation().toString()); - FileSystem fs = tableLocation.getFileSystem(SessionState.get().getConf()); + tableLocation = new Path(output.getTable().getDataLocation().toString()); + fs = tableLocation.getFileSystem(SessionState.get().getConf()); assert(fs.listStatus(tableLocation).length == 0); } } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java index 606208c..f2621ba 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java @@ -403,14 +403,16 @@ public Writable serialize(Object obj, ObjectInspector objInspector) serializedSize = 0; // Serialize each field + ObjectInspector foi; + Object f; for (int i = 0; i < fields.size(); i++) { // Append the separator if needed. if (i > 0) { serializeStream.write(serdeParams.separators[0]); } // Get the field objectInspector and the field object. - ObjectInspector foi = fields.get(i).getFieldObjectInspector(); - Object f = (list == null ? null : list.get(i)); + foi = fields.get(i).getFieldObjectInspector(); + f = (list == null ? null : list.get(i)); if (declaredFields != null && i >= declaredFields.size()) { throw new SerDeException("Error: expecting " + declaredFields.size() diff --git a/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRow.java b/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRow.java index 82b2ae9..c7bd5bd 100644 --- a/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRow.java +++ b/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRow.java @@ -340,6 +340,7 @@ public TRowStandardScheme getScheme() { public void read(org.apache.thrift.protocol.TProtocol iprot, TRow struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); + TColumnValue _elem94; // required while (true) { schemeField = iprot.readFieldBegin(); @@ -354,7 +355,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TRow struct) throws struct.colVals = new ArrayList(_list92.size); for (int _i93 = 0; _i93 < _list92.size; ++_i93) { - TColumnValue _elem94; // required _elem94 = new TColumnValue(); _elem94.read(iprot); struct.colVals.add(_elem94); diff --git a/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRowSet.java b/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRowSet.java index 2e5c5cc..b4a202c 100644 --- a/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRowSet.java +++ b/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TRowSet.java @@ -527,6 +527,8 @@ public TRowSetStandardScheme getScheme() { public void read(org.apache.thrift.protocol.TProtocol iprot, TRowSet struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); + TRow _elem102; // required + TColumn _elem105; // required while (true) { schemeField = iprot.readFieldBegin(); @@ -548,8 +550,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TRowSet struct) thr org.apache.thrift.protocol.TList _list100 = iprot.readListBegin(); struct.rows = new ArrayList(_list100.size); for (int _i101 = 0; _i101 < _list100.size; ++_i101) - { - TRow _elem102; // required + { _elem102 = new TRow(); _elem102.read(iprot); struct.rows.add(_elem102); @@ -568,7 +569,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TRowSet struct) thr struct.columns = new ArrayList(_list103.size); for (int _i104 = 0; _i104 < _list103.size; ++_i104) { - TColumn _elem105; // required _elem105 = new TColumn(); _elem105.read(iprot); struct.columns.add(_elem105); @@ -671,9 +671,9 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TRowSet struct) thro { org.apache.thrift.protocol.TList _list110 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); struct.rows = new ArrayList(_list110.size); + TRow _elem112; // required for (int _i111 = 0; _i111 < _list110.size; ++_i111) { - TRow _elem112; // required _elem112 = new TRow(); _elem112.read(iprot); struct.rows.add(_elem112); @@ -685,9 +685,9 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TRowSet struct) thro { org.apache.thrift.protocol.TList _list113 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); struct.columns = new ArrayList(_list113.size); + TColumn _elem115; // required for (int _i114 = 0; _i114 < _list113.size; ++_i114) { - TColumn _elem115; // required _elem115 = new TColumn(); _elem115.read(iprot); struct.columns.add(_elem115); diff --git a/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTableSchema.java b/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTableSchema.java index 23238ad..ec97445 100644 --- a/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTableSchema.java +++ b/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TTableSchema.java @@ -340,6 +340,7 @@ public TTableSchemaStandardScheme getScheme() { public void read(org.apache.thrift.protocol.TProtocol iprot, TTableSchema struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); + TColumnDesc _elem30; // required while (true) { schemeField = iprot.readFieldBegin(); @@ -354,7 +355,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, TTableSchema struct struct.columns = new ArrayList(_list28.size); for (int _i29 = 0; _i29 < _list28.size; ++_i29) { - TColumnDesc _elem30; // required _elem30 = new TColumnDesc(); _elem30.read(iprot); struct.columns.add(_elem30); @@ -423,9 +423,9 @@ public void read(org.apache.thrift.protocol.TProtocol prot, TTableSchema struct) { org.apache.thrift.protocol.TList _list33 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32()); struct.columns = new ArrayList(_list33.size); + TColumnDesc _elem35; // required for (int _i34 = 0; _i34 < _list33.size; ++_i34) { - TColumnDesc _elem35; // required _elem35 = new TColumnDesc(); _elem35.read(iprot); struct.columns.add(_elem35); diff --git a/shims/src/0.20/java/org/apache/hadoop/hive/shims/HiveHarFileSystem.java b/shims/src/0.20/java/org/apache/hadoop/hive/shims/HiveHarFileSystem.java index 323ebbb..ab1ca40 100644 --- a/shims/src/0.20/java/org/apache/hadoop/hive/shims/HiveHarFileSystem.java +++ b/shims/src/0.20/java/org/apache/hadoop/hive/shims/HiveHarFileSystem.java @@ -54,8 +54,9 @@ public ContentSummary getContentSummary(Path f) throws IOException { } // f is a directory long[] summary = {0, 0, 1}; + ContentSummary c; for(FileStatus s : listStatus(f)) { - ContentSummary c = s.isDir() ? getContentSummary(s.getPath()) : + c = s.isDir() ? getContentSummary(s.getPath()) : new ContentSummary(s.getLen(), 1, 0); summary[0] += c.getLength(); summary[1] += c.getFileCount(); diff --git a/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java b/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java index 8683496..148980f 100644 --- a/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java +++ b/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java @@ -161,10 +161,11 @@ public static String ensurePath(ZooKeeper zk, String path, List acl) throws InterruptedException { String[] pathComps = StringUtils.splitByWholeSeparator(path, "/"); String currentPath = ""; + String node; for (String pathComp : pathComps) { currentPath += "/" + pathComp; try { - String node = zk.create(currentPath, new byte[0], acl, + node = zk.create(currentPath, new byte[0], acl, CreateMode.PERSISTENT); LOGGER.info("Created path: " + node); } catch (KeeperException.NodeExistsException e) { @@ -213,19 +214,21 @@ public static int getPermFromString(String permString) { public static List parseACLs(String aclString) { String[] aclComps = StringUtils.splitByWholeSeparator(aclString, ","); List acl = new ArrayList(aclComps.length); + int firstColon, lastColon; + ACL newAcl; for (String a : aclComps) { if (StringUtils.isBlank(a)) { continue; } a = a.trim(); // from ZooKeeperMain private method - int firstColon = a.indexOf(':'); - int lastColon = a.lastIndexOf(':'); + firstColon = a.indexOf(':'); + lastColon = a.lastIndexOf(':'); if (firstColon == -1 || lastColon == -1 || firstColon == lastColon) { LOGGER.error(a + " does not have the form scheme:id:perm"); continue; } - ACL newAcl = new ACL(); + newAcl = new ACL(); newAcl.setId(new Id(a.substring(0, firstColon), a.substring( firstColon + 1, lastColon))); newAcl.setPerms(getPermFromString(a.substring(lastColon + 1))); @@ -287,8 +290,9 @@ public Configuration getConf() { ZooKeeper zk = getSession(); List nodes = zk.getChildren(masterKeyNode, false); Map result = new HashMap(); + byte[] data; for (String node : nodes) { - byte[] data = zk.getData(masterKeyNode + "/" + node, false, null); + data = zk.getData(masterKeyNode + "/" + node, false, null); if (data != null) { result.put(getSeq(node), data); }