From 033cf8fcf0f264ab4ebc6cfd63b12f39705b7221 Mon Sep 17 00:00:00 2001 From: Peter Somogyi Date: Mon, 28 Aug 2017 12:04:12 +0200 Subject: [PATCH] HBASE-18704 Upgrade hbase to commons-collections 4 Upgrade commons-collections:3.2.2 to commons-collections4:4.1 Add missing dependency for hbase-procedure, hbase-thrift Replace CircularFifoBuffer with CircularFifoQueue in WALProcedureStore and TaskMonitor --- hbase-common/pom.xml | 4 ++-- .../org/apache/hadoop/hbase/CompoundConfiguration.java | 6 +++--- hbase-procedure/pom.xml | 18 +++++++++++------- .../hbase/procedure2/store/wal/WALProcedureStore.java | 10 +++++----- hbase-server/pom.xml | 4 ++-- .../hadoop/hbase/io/hfile/bucket/BucketAllocator.java | 2 +- .../apache/hadoop/hbase/monitoring/TaskMonitor.java | 8 ++++---- .../hbase/regionserver/RegionCoprocessorHost.java | 10 +++++----- hbase-thrift/pom.xml | 4 ++++ .../apache/hadoop/hbase/thrift2/ThriftUtilities.java | 3 +-- .../hbase/thrift2/TestThriftHBaseServiceHandler.java | 7 ++----- pom.xml | 6 +++--- 12 files changed, 43 insertions(+), 39 deletions(-) diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml index 84e120b71e..9a9b7fddb2 100644 --- a/hbase-common/pom.xml +++ b/hbase-common/pom.xml @@ -238,8 +238,8 @@ compile - commons-collections - commons-collections + org.apache.commons + commons-collections4 compile diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CompoundConfiguration.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CompoundConfiguration.java index 5812cca5ca..2f97c6e388 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CompoundConfiguration.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CompoundConfiguration.java @@ -28,7 +28,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import org.apache.commons.collections.iterators.UnmodifiableIterator; +import org.apache.commons.collections4.iterators.UnmodifiableIterator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; @@ -388,7 +388,7 @@ public class CompoundConfiguration extends Configuration { } } - return UnmodifiableIterator.decorate(ret.entrySet().iterator()); + return UnmodifiableIterator.unmodifiableIterator(ret.entrySet().iterator()); } @Override @@ -419,4 +419,4 @@ public class CompoundConfiguration extends Configuration { public void writeXml(OutputStream out) throws IOException { throw new UnsupportedOperationException("Immutable Configuration"); } -}; +} diff --git a/hbase-procedure/pom.xml b/hbase-procedure/pom.xml index 67f769133e..21a43efb42 100644 --- a/hbase-procedure/pom.xml +++ b/hbase-procedure/pom.xml @@ -63,9 +63,9 @@ ${project.version} tests - - org.apache.hbase - hbase-annotations + + org.apache.hbase + hbase-annotations test-jar test @@ -89,10 +89,14 @@ commons-logging commons-logging - - org.apache.hbase - hbase-metrics-api - + + org.apache.hbase + hbase-metrics-api + + + org.apache.commons + commons-collections4 + diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java index ed191bb0f8..d6275eaaf0 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java @@ -36,7 +36,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; -import org.apache.commons.collections.buffer.CircularFifoBuffer; +import org.apache.commons.collections4.queue.CircularFifoQueue; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -155,7 +155,7 @@ public class WALProcedureStore extends ProcedureStoreBase { private int syncWaitMsec; // Variables used for UI display - private CircularFifoBuffer syncMetricsBuffer; + private CircularFifoQueue syncMetricsQueue; public static class SyncMetrics { private long timestamp; @@ -229,7 +229,7 @@ public class WALProcedureStore extends ProcedureStoreBase { useHsync = conf.getBoolean(USE_HSYNC_CONF_KEY, DEFAULT_USE_HSYNC); // WebUI - syncMetricsBuffer = new CircularFifoBuffer( + syncMetricsQueue = new CircularFifoQueue( conf.getInt(STORE_WAL_SYNC_STATS_COUNT, DEFAULT_SYNC_STATS_COUNT)); // Init sync thread @@ -777,7 +777,7 @@ public class WALProcedureStore extends ProcedureStoreBase { syncMetrics.syncedEntries = slotIndex; syncMetrics.totalSyncedBytes = totalSyncedToStore; syncMetrics.syncedPerSec = syncedPerSec; - syncMetricsBuffer.add(syncMetrics); + syncMetricsQueue.add(syncMetrics); // sync inSync.set(true); @@ -808,7 +808,7 @@ public class WALProcedureStore extends ProcedureStoreBase { public ArrayList getSyncMetrics() { lock.lock(); try { - return new ArrayList<>(syncMetricsBuffer); + return new ArrayList<>(syncMetricsQueue); } finally { lock.unlock(); } diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index e1ef88234f..3f45affe4e 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -423,8 +423,8 @@ commons-codec - commons-collections - commons-collections + org.apache.commons + commons-collections4 org.apache.hbase diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java index d1b8bb72cf..b4c5a449ce 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java @@ -30,7 +30,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hbase.shaded.com.google.common.collect.MinMaxPriorityQueue; -import org.apache.commons.collections.map.LinkedMap; +import org.apache.commons.collections4.map.LinkedMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java index ad9bd0225a..732265538e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/monitoring/TaskMonitor.java @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import org.apache.commons.collections.buffer.CircularFifoBuffer; +import org.apache.commons.collections4.queue.CircularFifoQueue; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -61,7 +61,7 @@ public class TaskMonitor { private final int maxTasks; private final long rpcWarnTime; private final long expirationTime; - private final CircularFifoBuffer tasks; + private final CircularFifoQueue tasks; private final List rpcTasks; private final long monitorInterval; private Thread monitorThread; @@ -70,7 +70,7 @@ public class TaskMonitor { maxTasks = conf.getInt(MAX_TASKS_KEY, DEFAULT_MAX_TASKS); expirationTime = conf.getLong(EXPIRATION_TIME_KEY, DEFAULT_EXPIRATION_TIME); rpcWarnTime = conf.getLong(RPC_WARN_TIME_KEY, DEFAULT_RPC_WARN_TIME); - tasks = new CircularFifoBuffer(maxTasks); + tasks = new CircularFifoQueue(maxTasks); rpcTasks = Lists.newArrayList(); monitorInterval = conf.getLong(MONITOR_INTERVAL_KEY, DEFAULT_MONITOR_INTERVAL); monitorThread = new Thread(new MonitorRunnable()); @@ -349,6 +349,6 @@ public class TaskMonitor { * @param task monitored task * @return false if a task is accepted, true if it is filtered */ - boolean filter(MonitoredTask t); + boolean filter(MonitoredTask task); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index dfe22ba545..e2a2622d66 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -1,4 +1,4 @@ -/* +/** * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -29,8 +29,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.regex.Matcher; -import org.apache.commons.collections.map.AbstractReferenceMap; -import org.apache.commons.collections.map.ReferenceMap; +import org.apache.commons.collections4.map.AbstractReferenceMap; +import org.apache.commons.collections4.map.ReferenceMap; import org.apache.commons.lang.ClassUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -43,7 +43,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Append; @@ -98,7 +97,8 @@ public class RegionCoprocessorHost private static final Log LOG = LogFactory.getLog(RegionCoprocessorHost.class); // The shared data map private static ReferenceMap sharedDataMap = - new ReferenceMap(AbstractReferenceMap.HARD, AbstractReferenceMap.WEAK); + new ReferenceMap(AbstractReferenceMap.ReferenceStrength.HARD, + AbstractReferenceMap.ReferenceStrength.WEAK); // optimization: no need to call postScannerFilterRow, if no coprocessor implements it private final boolean hasCustomPostScannerFilterRow; diff --git a/hbase-thrift/pom.xml b/hbase-thrift/pom.xml index 9d7a4b5b03..d6f77c3abf 100644 --- a/hbase-thrift/pom.xml +++ b/hbase-thrift/pom.xml @@ -239,6 +239,10 @@ commons-math + org.apache.commons + commons-collections4 + + commons-lang commons-lang diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java index f318f8e6be..44062bb3fc 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java @@ -26,7 +26,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.ServerName; @@ -55,7 +55,6 @@ import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement; import org.apache.hadoop.hbase.thrift2.generated.TColumnValue; import org.apache.hadoop.hbase.thrift2.generated.TCompareOp; import org.apache.hadoop.hbase.thrift2.generated.TDelete; -import org.apache.hadoop.hbase.thrift2.generated.TDeleteType; import org.apache.hadoop.hbase.thrift2.generated.TDurability; import org.apache.hadoop.hbase.thrift2.generated.TGet; import org.apache.hadoop.hbase.thrift2.generated.THRegionInfo; diff --git a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java index 7a35d29300..d7757ef216 100644 --- a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java +++ b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java @@ -1,4 +1,4 @@ -/* +/** * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -19,7 +19,7 @@ package org.apache.hadoop.hbase.thrift2; import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -31,11 +31,8 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.Delete; diff --git a/pom.xml b/pom.xml index 370166bc82..9d522bbd33 100755 --- a/pom.xml +++ b/pom.xml @@ -1387,7 +1387,7 @@ 2.2 3.3.6 - 3.2.2 + 4.1 4.5.3 4.4.6 3.2.1 @@ -1765,8 +1765,8 @@ ${metrics-core.version} - commons-collections - commons-collections + org.apache.commons + commons-collections4 ${collections.version} -- 2.13.1