Index: dev-support/findbugs-exclude.xml
===================================================================
--- dev-support/findbugs-exclude.xml (revision 1554150)
+++ dev-support/findbugs-exclude.xml (working copy)
@@ -37,6 +37,10 @@
+
+
+
+
Index: hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
===================================================================
--- hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (revision 1554150)
+++ hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (working copy)
@@ -159,11 +159,11 @@
private volatile Configuration conf;
private final long pause;
- private final int numRetries;
+ protected final int numRetries;
// Some operations can take a long time such as disable of big table.
// numRetries is for 'normal' stuff... Multiply by this factor when
// want to wait a long time.
- private final int retryLongerMultiplier;
+ protected final int retryLongerMultiplier;
private boolean aborted;
private boolean cleanupConnectionOnClose = false; // close the connection in close()
private boolean closed = false;
@@ -394,7 +394,7 @@
return getTableDescriptor(TableName.valueOf(tableName));
}
- private long getPauseTime(int tries) {
+ protected long getPauseTime(int tries) {
int triesCount = tries;
if (triesCount >= HConstants.RETRY_BACKOFF.length) {
triesCount = HConstants.RETRY_BACKOFF.length - 1;
Index: hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ByteArrayComparable.java
===================================================================
--- hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ByteArrayComparable.java (revision 1554150)
+++ hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ByteArrayComparable.java (working copy)
@@ -32,7 +32,7 @@
@InterfaceStability.Stable
public abstract class ByteArrayComparable implements Comparable {
- byte[] value;
+ protected byte[] value;
/**
* Constructor.
Index: hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
===================================================================
--- hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java (revision 1554150)
+++ hbase-client/src/main/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java (working copy)
@@ -303,7 +303,7 @@
return filter;
}
- FilterProtos.SingleColumnValueFilter convert() {
+ public FilterProtos.SingleColumnValueFilter convert() {
FilterProtos.SingleColumnValueFilter.Builder builder =
FilterProtos.SingleColumnValueFilter.newBuilder();
if (this.columnFamily != null) {
Index: hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java
===================================================================
--- hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java (revision 1554150)
+++ hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTableReadOnly.java (working copy)
@@ -26,11 +26,15 @@
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
+import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table.State;
import org.apache.zookeeper.KeeperException;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import java.util.Map.Entry;
/**
* Non-instantiable class that provides helper functions for
@@ -132,7 +136,31 @@
}
return disabledTables;
}
+
+ /**
+ * Gets all the tables by state.
+ * @return map of state vs tables.
+ * @throws KeeperException
+ */
+ public static Map>
+ getTableNamesByState(ZooKeeperWatcher zkw) throws KeeperException {
+ Map> tablesByState = new HashMap>();
+ List children = ZKUtil.listChildrenNoWatch(zkw, zkw.tableZNode);
+ for (String child : children) {
+ TableName tableName = TableName.valueOf(child);
+ ZooKeeperProtos.Table.State state = getTableState(zkw, tableName);
+ if (tablesByState.containsKey(state)) {
+ tablesByState.get(state).add(tableName);
+ } else {
+ HashSet set = new HashSet();
+ set.add(tableName);
+ tablesByState.put(state, set);
+ }
+ }
+ return tablesByState;
+ }
+
static boolean isTableState(final ZooKeeperProtos.Table.State expectedState,
final ZooKeeperProtos.Table.State currentState) {
return currentState != null && currentState.equals(expectedState);
Index: hbase-common/src/main/resources/hbase-default.xml
===================================================================
--- hbase-common/src/main/resources/hbase-default.xml (revision 1554150)
+++ hbase-common/src/main/resources/hbase-default.xml (working copy)
@@ -1112,4 +1112,23 @@
as the SimpleLoadBalancer).
+
+ hbase.use.secondary.index
+ false
+ Enable this property when you are using secondary index.
+
+
+
+ hbase.index.half.storefile.reader.class
+ org.apache.hadoop.hbase.index.io.IndexHalfStoreFileReader
+
+ Do not change this. It is used internally when secondary
+ index is enabled
+
+
+
+ hbase.index.loadbalancer.delegator.class
+ org.apache.hadoop.hbase.master.balancer.StochasticLoadBalancer
+
+
Index: hbase-protocol/src/main/protobuf/ValuePartition.proto
===================================================================
--- hbase-protocol/src/main/protobuf/ValuePartition.proto (revision 0)
+++ hbase-protocol/src/main/protobuf/ValuePartition.proto (working copy)
@@ -0,0 +1,37 @@
+
+option java_package = "org.apache.hadoop.hbase.index.protobuf.generated";
+option java_outer_classname = "ValuePartitionProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "Filter.proto";
+
+message ValuePartition {
+ extensions 100 to 199;
+ enum PartitionType{
+ SEPARATOR = 0;
+ SPATIAL = 1;
+ NONE = 2;
+ }
+ optional PartitionType partition_type = 1 [default = NONE];
+}
+
+message SpatialPartition {
+ extend ValuePartition {
+ required int32 offset = 100;
+ required int32 length = 101;
+ }
+}
+
+message SeparatorPartition {
+ extend ValuePartition {
+ required bytes separator = 102;
+ required int32 position = 103;
+ }
+}
+
+message SingleColumnValuePartitionFilter {
+ required SingleColumnValueFilter single_column_value_filter = 1;
+ required ValuePartition value_partition = 2;
+}
Index: hbase-secondaryindex/pom.xml
===================================================================
--- hbase-secondaryindex/pom.xml (revision 0)
+++ hbase-secondaryindex/pom.xml (working copy)
@@ -0,0 +1,279 @@
+
+
+
+ 4.0.0
+
+ hbase
+ org.apache.hbase
+ 0.99.0-SNAPSHOT
+ ..
+
+
+ hbase-secondaryindex
+ HBase - Secondary Index
+ Secondary index related funcationality for HBase
+
+
+
+
+ org.apache.maven.plugins
+ maven-site-plugin
+
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+
+
+
+ maven-assembly-plugin
+ ${maven.assembly.version}
+
+ true
+
+
+
+ maven-surefire-plugin
+
+
+
+ secondPartTestsExecution
+ test
+
+ test
+
+
+ true
+
+
+
+
+
+
+
+
+
+
+ org.apache.hbase
+ hbase-common
+
+
+ org.apache.hbase
+ hbase-client
+
+
+ org.apache.hbase
+ hbase-server
+
+
+ org.apache.hbase
+ hbase-common
+ test-jar
+
+
+ org.apache.hbase
+ hbase-server
+ test-jar
+
+
+ org.apache.hbase
+ hbase-hadoop-compat
+
+
+ org.apache.hbase
+ hbase-hadoop-compat
+ test-jar
+ test
+
+
+ org.apache.hbase
+ ${compat.module}
+ ${project.version}
+
+
+ org.apache.hbase
+ ${compat.module}
+ ${project.version}
+ test-jar
+ test
+
+
+
+ commons-lang
+ commons-lang
+
+
+ commons-logging
+ commons-logging
+
+
+
+
+
+
+ skipSecondaryIndexTests
+
+
+ skipSecondaryIndexTests
+
+
+
+ true
+ true
+
+
+
+
+ hadoop-1.1
+
+
+
+ hadoop.profile
+ 1.1
+
+
+
+
+ org.apache.hadoop
+ hadoop-core
+
+
+ org.apache.hadoop
+ hadoop-test
+
+
+
+
+
+
+ hadoop-1.0
+
+
+ hadoop.profile
+ 1.0
+
+
+
+
+ org.apache.hadoop
+ hadoop-core
+
+
+ org.apache.hadoop
+ hadoop-test
+
+
+
+
+
+
+ hadoop-2.0
+
+
+
+ !hadoop.profile
+
+
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+ org.apache.hadoop
+ hadoop-common
+ ${hadoop-two.version}
+ test-jar
+
+
+ org.apache.hadoop
+ hadoop-auth
+
+
+ org.apache.hadoop
+ hadoop-client
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-jobclient
+ test-jar
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ test-jar
+
+
+ org.apache.hadoop
+ hadoop-annotations
+
+
+ org.apache.hadoop
+ hadoop-minicluster
+ test
+
+
+
+
+
+
+ hadoop-3.0
+
+
+ hadoop.profile
+ 3.0
+
+
+
+ 3.0-SNAPSHOT
+
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+
+
+
+
+
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/EqualsExpression.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/EqualsExpression.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/EqualsExpression.java (working copy)
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.client;
+
+import java.io.Serializable;
+
+import org.apache.hadoop.hbase.index.Column;
+
+/**
+ * Can be used to specify an equals condition on a column associated with an index.
+ */
+public class EqualsExpression implements Serializable {
+
+ private static final long serialVersionUID = -7130697408286943018L;
+
+ private Column column;
+ private byte[] value;
+
+ public EqualsExpression(Column column, byte[] value) {
+ this.column = column;
+ this.value = value;
+ }
+
+ public Column getColumn() {
+ return this.column;
+ }
+
+ public byte[] getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return "EqualsExpression : Column[" + this.column + ']';
+ }
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexAdmin.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexAdmin.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexAdmin.java (working copy)
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.client;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.SocketTimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotEnabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.MetaScanner;
+import org.apache.hadoop.hbase.client.RegionOfflineException;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
+import org.apache.hadoop.hbase.index.Constants;
+import org.apache.hadoop.hbase.index.util.IndexUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Extension of HBaseAdmin to perform index related operations. This also should be used when we
+ * create table with index details, and other admin operations on indexed table.
+ */
+public class IndexAdmin extends HBaseAdmin {
+ private final Log LOG = LogFactory.getLog(this.getClass().getName());
+
+ public IndexAdmin(Configuration c) throws IOException {
+ super(c);
+ }
+
+ public IndexAdmin(HConnection connection) throws MasterNotRunningException,
+ ZooKeeperConnectionException {
+ super(connection);
+ }
+
+ @Override
+ public void createTable(final HTableDescriptor desc, byte[][] splitKeys) throws IOException {
+ try {
+ createTableAsync(desc, splitKeys);
+ } catch (SocketTimeoutException ste) {
+ LOG.warn("Creating " + desc.getNameAsString() + " took too long", ste);
+ }
+ int numRegs = splitKeys == null ? 1 : splitKeys.length + 1;
+ int prevRegCount = 0;
+
+ MetaScannerVisitorBaseWithTableName userTableVisitor = null;
+ MetaScannerVisitorBaseWithTableName indexTableVisitor = null;
+ boolean indexedHTD = desc.getValue(Constants.INDEX_SPEC_KEY) != null;
+
+ for (int tries = 0; tries < this.numRetries * this.retryLongerMultiplier; ++tries) {
+
+ AtomicInteger actualRegCount = null;
+ // Wait for new table to come on-line
+ if (userTableVisitor == null) {
+ userTableVisitor = new MetaScannerVisitorBaseWithTableName(desc.getNameAsString());
+ }
+ actualRegCount = userTableVisitor.getActualRgnCnt();
+ actualRegCount.set(0);
+ MetaScanner.metaScan(getConfiguration(), getConnection(), userTableVisitor,
+ desc.getTableName());
+ if (actualRegCount.get() != numRegs) {
+ if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
+ throw new RegionOfflineException("Only " + actualRegCount.get() + " of " + numRegs
+ + " regions are online; retries exhausted.");
+ }
+ try { // Sleep
+ Thread.sleep(getPauseTime(tries));
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException("Interrupted when opening" + " regions; "
+ + actualRegCount.get() + " of " + numRegs + " regions processed so far");
+ }
+ if (actualRegCount.get() > prevRegCount) { // Making progress
+ prevRegCount = actualRegCount.get();
+ tries = -1;
+ }
+ } else {
+ if (indexedHTD) {
+ TableName indexTableName =
+ TableName.valueOf(IndexUtils.getIndexTableName(desc.getName()));
+ if (indexTableVisitor == null) {
+ indexTableVisitor =
+ new MetaScannerVisitorBaseWithTableName(indexTableName.getNameAsString());
+ }
+ actualRegCount = indexTableVisitor.getActualRgnCnt();
+ actualRegCount.set(0);
+ MetaScanner.metaScan(getConfiguration(), getConnection(), indexTableVisitor,
+ indexTableName);
+ if (actualRegCount.get() != numRegs) {
+ if (tries == this.numRetries * this.retryLongerMultiplier - 1) {
+ throw new RegionOfflineException("Only " + actualRegCount.get() + " of " + numRegs
+ + " regions are online; retries exhausted.");
+ }
+ try { // Sleep
+ Thread.sleep(getPauseTime(tries));
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException("Interrupted when opening" + " regions; "
+ + actualRegCount.get() + " of " + numRegs + " regions processed so far");
+ }
+ if (actualRegCount.get() > prevRegCount) { // Making progress
+ prevRegCount = actualRegCount.get();
+ tries = -1;
+ }
+ } else if (isTableEnabled(indexTableName)) {
+ return;
+ }
+ } else if (isTableEnabled(desc.getName())) {
+ return;
+ }
+ }
+ }
+ throw new TableNotEnabledException("Retries exhausted while still waiting for table: "
+ + desc.getNameAsString() + " to be enabled");
+ }
+
+ static class MetaScannerVisitorBaseWithTableName implements MetaScannerVisitor {
+ byte[] tableName = null;
+ AtomicInteger actualRegCount = new AtomicInteger(0);
+
+ MetaScannerVisitorBaseWithTableName(String tableName) {
+ this.tableName = Bytes.toBytes(tableName);
+ }
+
+ AtomicInteger getActualRgnCnt() {
+ return actualRegCount;
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public boolean processRow(Result rowResult) throws IOException {
+ HRegionInfo info = HRegionInfo.getHRegionInfo(rowResult);
+ // If regioninfo is null, skip this row
+ if (info == null) {
+ return true;
+ }
+ if (!(Bytes.equals(info.getTable().getName(), tableName))) {
+ return false;
+ }
+ ServerName serverName = HRegionInfo.getServerName(rowResult);
+ // Make sure that regions are assigned to server
+ if (!(info.isOffline() || info.isSplit()) && serverName != null
+ && serverName.getHostAndPort() != null) {
+ actualRegCount.incrementAndGet();
+ }
+ return true;
+ }
+ }
+
+ @Override
+ public boolean isTableEnabled(TableName tableName) throws IOException {
+ if (!tableExists(tableName)) {
+ throw new TableNotFoundException(tableName);
+ }
+ boolean indexEnabled = getConfiguration().getBoolean("hbase.use.secondary.index", false);
+ if (!indexEnabled) {
+ return getConnection().isTableEnabled(tableName);
+ } else {
+ boolean isTableEnabled = getConnection().isTableEnabled(tableName);
+ if (isTableEnabled && !IndexUtils.isIndexTable(tableName.getNameAsString())) {
+ TableName indexTableName =
+ TableName.valueOf(IndexUtils.getIndexTableName(tableName.getNameAsString()));
+ if (getConnection().isTableAvailable(indexTableName)) {
+ return getConnection().isTableEnabled(indexTableName);
+ }
+ return true;
+ }
+ return isTableEnabled;
+ }
+ }
+
+ @Override
+ public boolean isTableDisabled(TableName tableName) throws IOException {
+ if (!tableExists(tableName)) {
+ throw new TableNotFoundException(tableName);
+ }
+ boolean indexEnabled = getConfiguration().getBoolean("hbase.use.secondary.index", false);
+ if (!indexEnabled) {
+ return getConnection().isTableDisabled(tableName);
+ } else {
+ boolean isTableDisabled = getConnection().isTableDisabled(tableName);
+ if (isTableDisabled && !IndexUtils.isIndexTable(tableName.getNameAsString())) {
+ TableName indexTableName =
+ TableName.valueOf(IndexUtils.getIndexTableName(tableName.getNameAsString()));
+ if (getConnection().isTableAvailable(indexTableName)) {
+ return getConnection().isTableDisabled(indexTableName);
+ }
+ return true;
+ }
+ return isTableDisabled;
+ }
+ }
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexExpression.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexExpression.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexExpression.java (working copy)
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.client;
+
+import java.io.Serializable;
+
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.index.Constants;
+
+/**
+ * While scan the index(s) to be used can be explicitly passed from client application.
+ * Objects of IndexExpression can specify the index(s) details.
+ *
+ * @see SingleIndexExpression
+ * @see MultiIndexExpression
+ * @see NoIndexExpression
+ * @see Scan#setAttribute(String, byte[])
+ * @see Constants#INDEX_EXPRESSION
+ * @see IndexUtils#toBytes(IndexExpression)
+ */
+public interface IndexExpression extends Serializable {
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexUtils.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexUtils.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/IndexUtils.java (working copy)
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.client;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+import org.apache.hadoop.hbase.client.Scan;
+
+/**
+ * Client side utility class for using Secondary Index.
+ */
+public class IndexUtils {
+
+ private IndexUtils() {
+ // Avoid instantiation of this class.
+ }
+
+ /**
+ * Utility to convert IndexExpression into byte[]. Can be used to pass the IndexExpression in the
+ * Scan attributes.
+ * @see Scan#setAttribute(String, byte[])
+ * @param indexExpression
+ * @return byte[]
+ * @throws IOException
+ */
+ public static byte[] toBytes(IndexExpression indexExpression) throws IOException {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(bos);
+ oos.writeObject(indexExpression);
+ return bos.toByteArray();
+ }
+
+ /**
+ * Creates back IndexExpression from byte[]
+ * @param bytes
+ * @return IndexExpression deserialized from the specified bytes.
+ * @throws IOException
+ * @throws ClassNotFoundException
+ */
+ public static IndexExpression toIndexExpression(byte[] bytes) throws IOException,
+ ClassNotFoundException {
+ ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
+ ObjectInputStream ois = new ObjectInputStream(bis);
+ return (IndexExpression) ois.readObject();
+ }
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/MultiIndexExpression.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/MultiIndexExpression.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/MultiIndexExpression.java (working copy)
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.client;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.index.Constants;
+import org.apache.hadoop.hbase.index.GroupingCondition;
+
+/**
+ * Can be used to group a set of {@link SingleIndexExpression} with AND/OR {@link GroupingCondition}
+ *
+ * @see Scan#setAttribute(String, byte[])
+ * @see Constants#INDEX_EXPRESSION
+ * @see IndexUtils#toBytes(IndexExpression)
+ */
+public class MultiIndexExpression implements IndexExpression {
+
+ private static final long serialVersionUID = 5322668904124942100L;
+
+ private List indexExpressions = new ArrayList();
+
+ private GroupingCondition groupingCondition;
+
+ public MultiIndexExpression(GroupingCondition condition) {
+ this.groupingCondition = condition;
+ }
+
+ public GroupingCondition getGroupingCondition() {
+ return this.groupingCondition;
+ }
+
+ public void addIndexExpression(IndexExpression indexExpression) {
+ this.indexExpressions.add(indexExpression);
+ }
+
+ public List getIndexExpressions() {
+ return this.indexExpressions;
+ }
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/NoIndexExpression.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/NoIndexExpression.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/NoIndexExpression.java (working copy)
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.client;
+
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.index.Constants;
+
+/**
+ * Pass this as the index expression via Scan attribute when a scan query should not use any index
+ * which is possible to be used
+ *
+ * @see Scan#setAttribute(String, byte[])
+ * @see Constants#INDEX_EXPRESSION
+ * @see IndexUtils#toBytes(IndexExpression)
+ */
+public class NoIndexExpression implements IndexExpression {
+
+ private static final long serialVersionUID = 5445463806972787598L;
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/RangeExpression.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/RangeExpression.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/RangeExpression.java (working copy)
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.client;
+
+import java.io.Serializable;
+
+import org.apache.hadoop.hbase.index.Column;
+
+/**
+ * Can be used to specify a range condition on a column associated with an index. When the range is
+ * non closed at one end (to specific upper bound but only lower bound) pass the corresponding bound
+ * value as null.
+ */
+public class RangeExpression implements Serializable {
+
+ private static final long serialVersionUID = 8772267632040419734L;
+
+ private Column column;
+ private byte[] lowerBoundValue;
+ private byte[] upperBoundValue;
+ private boolean lowerBoundInclusive;
+ private boolean upperBoundInclusive;
+
+ public Column getColumn() {
+ return column;
+ }
+
+ public byte[] getLowerBoundValue() {
+ return lowerBoundValue;
+ }
+
+ public byte[] getUpperBoundValue() {
+ return upperBoundValue;
+ }
+
+ public boolean isLowerBoundInclusive() {
+ return lowerBoundInclusive;
+ }
+
+ public boolean isUpperBoundInclusive() {
+ return upperBoundInclusive;
+ }
+
+ /**
+ * When the range is non closed at one end (to specific upper bound but only lower bound) pass the
+ * corresponding bound value as null.
+ * @param column
+ * @param lowerBoundValue
+ * @param upperBoundValue
+ * @param lowerBoundInclusive
+ * @param upperBoundInclusive
+ */
+ public RangeExpression(Column column, byte[] lowerBoundValue, byte[] upperBoundValue,
+ boolean lowerBoundInclusive, boolean upperBoundInclusive) {
+ if (column == null || (lowerBoundValue == null && upperBoundValue == null)) {
+ throw new IllegalArgumentException();
+ }
+ this.column = column;
+ this.lowerBoundValue = lowerBoundValue;
+ this.upperBoundValue = upperBoundValue;
+ this.lowerBoundInclusive = lowerBoundInclusive;
+ this.upperBoundInclusive = upperBoundInclusive;
+ }
+
+ @Override
+ public String toString() {
+ return "RangeExpression : Column[" + this.column + "], lowerBoundInclusive : "
+ + this.lowerBoundInclusive + ", upperBoundInclusive : " + this.upperBoundInclusive;
+ }
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/SingleIndexExpression.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/SingleIndexExpression.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/client/SingleIndexExpression.java (working copy)
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.client;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.index.Constants;
+
+/**
+ * This can specify one index details to be used. The index can be single column or multi column
+ * index. This index can be used for equals or range condition on columns. For a multi column index,
+ * there can be 0 or more equals condition associated with this index in the scan and at max one
+ * range condition (on a column). For an index the columns are in an order. There can not be any
+ * equals condition specified on columns coming after the range conditioned column as per the
+ * columns order in the index specification.
+ *
+ * @see Scan#setAttribute(String, byte[])
+ * @see Constants#INDEX_EXPRESSION
+ * @see IndexUtils#toBytes(IndexExpression)
+ */
+public class SingleIndexExpression implements IndexExpression {
+
+ private static final long serialVersionUID = 893160134306193043L;
+
+ private String indexName;
+
+ private List equalsExpressions = new ArrayList();
+
+ private RangeExpression rangeExpression;
+
+ public SingleIndexExpression(String indexName) {
+ this.indexName = indexName;
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+
+ /**
+ * This is expected to be called in the order of columns specified in the Index. If index is on
+ * columns cf1:c1, cf1:c2 and cf2:c3 when creating the SingleIndexExpression call this method in
+ * the same order as of above
+ * @param equalsExpression
+ */
+ public void addEqualsExpression(EqualsExpression equalsExpression) {
+ this.equalsExpressions.add(equalsExpression);
+ }
+
+ public List getEqualsExpressions() {
+ return this.equalsExpressions;
+ }
+
+ public void setRangeExpression(RangeExpression rangeExpression) {
+ this.rangeExpression = rangeExpression;
+ }
+
+ public RangeExpression getRangeExpression() {
+ return this.rangeExpression;
+ }
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Column.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Column.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Column.java (working copy)
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index;
+
+import java.io.Serializable;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class Column implements Serializable {
+ private static final long serialVersionUID = -1958705310924323448L;
+
+ private byte[] cf;
+ private byte[] qualifier;
+ private ValuePartition valuePartition = null;
+
+ public Column() {
+ }
+
+ public Column(byte[] cf, byte[] qualifier) {
+ this.cf = cf;
+ this.qualifier = qualifier;
+ }
+
+ public Column(byte[] cf, byte[] qualifier, ValuePartition vp) {
+ this.cf = cf;
+ this.qualifier = qualifier;
+ this.valuePartition = vp;
+ }
+
+ public void setFamily(byte[] cf) {
+ this.cf = cf;
+ }
+
+ public void setQualifier(byte[] qualifier) {
+ this.qualifier = qualifier;
+ }
+
+ public byte[] getFamily() {
+ return cf;
+ }
+
+ public byte[] getQualifier() {
+ return qualifier;
+ }
+
+ public ValuePartition getValuePartition() {
+ return this.valuePartition;
+ }
+
+ public void setValuePartition(ValuePartition vp) {
+ this.valuePartition = vp;
+ }
+
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Column)) return false;
+ Column that = (Column) obj;
+ if (!(Bytes.equals(this.cf, that.cf))) return false;
+ if (!(Bytes.equals(this.qualifier, that.qualifier))) return false;
+ if (valuePartition == null && that.valuePartition == null) {
+ return true;
+ } else if (valuePartition != null && that.valuePartition != null) {
+ return valuePartition.equals(that.valuePartition);
+ } else {
+ return false;
+ }
+ }
+
+ public int hashCode() {
+ int result = Bytes.hashCode(this.cf);
+ result ^= Bytes.hashCode(this.qualifier);
+ if (valuePartition != null) result ^= valuePartition.hashCode();
+ return result;
+ }
+
+ public String toString() {
+ return Bytes.toString(this.cf) + " : " + Bytes.toString(this.qualifier);
+ }
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ColumnQualifier.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ColumnQualifier.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/ColumnQualifier.java (working copy)
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.index.protobuf.generated.ValuePartitionProtos.ValuePartition.PartitionType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ *
+ * A ColumnQualifier contains information about column family name,qualifier and maximum value
+ * length used in index specification. maValueLength is used to make sure the common pattern for all
+ * the rowkeys in the index table.
+ *
+ * ColumnQualifier is used as input for index specification to specify column family and column
+ * qualifier on which we are going to do indexing.
+ *
+ */
+public class ColumnQualifier implements WritableComparable {
+
+ private byte[] cfBytes;
+
+ private byte[] qualifierBytes;
+
+ private int maxValueLength;
+
+ private ValuePartition valuePartition = null;
+
+ private ValueType type;
+
+ public ColumnQualifier() {
+ // Dummy constructor which is needed for the readFields
+ }
+
+ public ColumnQualifier(String cf, String qualifier) {
+ this(Bytes.toBytes(cf), Bytes.toBytes(qualifier));
+ }
+
+ public ColumnQualifier(byte[] cf, byte[] qualifier) {
+ this(cf, qualifier, ValueType.String, 0, null);
+ }
+
+ public ColumnQualifier(String cf, String qualifier, ValueType type, int maxValueLength) {
+ this(Bytes.toBytes(cf), Bytes.toBytes(qualifier), type, maxValueLength, null);
+ }
+
+ public ColumnQualifier(String cf, String qualifier, ValueType type, int maxValueLength,
+ ValuePartition vp) {
+ this(Bytes.toBytes(cf), Bytes.toBytes(qualifier), type, maxValueLength, vp);
+ }
+
+ public ColumnQualifier(byte[] cf, byte[] qualifier, ValueType type, int maxValueLength,
+ ValuePartition vp) {
+ this.cfBytes = cf;
+ this.qualifierBytes = qualifier;
+ this.type = type;
+ this.maxValueLength = maxValueLength;
+ this.valuePartition = vp;
+ }
+
+ /**
+ * @return Column Family as string
+ */
+ public String getColumnFamilyString() {
+ return Bytes.toString(this.cfBytes);
+ }
+
+ /**
+ * @return Column qualifier as string
+ */
+ public String getQualifierString() {
+ return Bytes.toString(this.qualifierBytes);
+ }
+
+ /**
+ * @return Column family as byte array
+ */
+ public byte[] getColumnFamily() {
+ return this.cfBytes;
+ }
+
+ /**
+ * @return Column qualifier as byte array
+ */
+ public byte[] getQualifier() {
+ return this.qualifierBytes;
+ }
+
+ public ValuePartition getValuePartition() {
+ return valuePartition;
+ }
+
+ /**
+ * @param in input stream
+ * @throws IOException
+ */
+ public void readFields(DataInput in) throws IOException {
+ this.cfBytes = Bytes.readByteArray(in);
+ this.qualifierBytes = Bytes.readByteArray(in);
+ this.type = ValueType.valueOf(Bytes.toString(Bytes.readByteArray(in)));
+ this.maxValueLength = in.readInt();
+ PartitionType p = PartitionType.valueOf(in.readUTF());
+ try {
+ if (p.equals(PartitionType.SEPARATOR)) {
+ this.valuePartition = SeparatorPartition.parseFrom(Bytes.readByteArray(in));
+ } else if (p.equals(PartitionType.SPATIAL)) {
+ this.valuePartition = SpatialPartition.parseFrom(Bytes.readByteArray(in));
+ }
+ } catch (DeserializationException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * @param out output stream
+ * @throws IOException
+ */
+ public void write(DataOutput out) throws IOException {
+ Bytes.writeByteArray(out, this.cfBytes);
+ Bytes.writeByteArray(out, this.qualifierBytes);
+ Bytes.writeByteArray(out, Bytes.toBytes(this.type.name()));
+ out.writeInt(maxValueLength);
+ if (this.valuePartition == null) {
+ out.writeUTF(PartitionType.NONE.name());
+ } else {
+ out.writeUTF(valuePartition.getPartitionType().name());
+ Bytes.writeByteArray(out, this.valuePartition.toByteArray());
+ }
+ }
+
+ /**
+ * @param cq with whom to compare
+ * @return return true if both objects are equal otherwise false
+ */
+ @Override
+ public boolean equals(Object cq) {
+ if (this == cq) {
+ return true;
+ }
+ if (false == (cq instanceof ColumnQualifier)) {
+ return false;
+ }
+ return this.compareTo((ColumnQualifier) cq) == 0;
+ }
+
+ /**
+ * return hashcode of object
+ */
+ public int hashCode() {
+ int result = Bytes.hashCode(this.cfBytes);
+ result ^= Bytes.hashCode(this.qualifierBytes);
+ result ^= this.maxValueLength;
+ if (valuePartition != null) result ^= valuePartition.hashCode();
+ return result;
+ }
+
+ /**
+ * @param cq with whom to compare
+ * @return int
+ */
+ @Override
+ public int compareTo(ColumnQualifier cq) {
+ int diff = 0;
+ diff = Bytes.compareTo(this.cfBytes, cq.cfBytes);
+ if (0 == diff) {
+ diff = Bytes.compareTo(this.qualifierBytes, cq.qualifierBytes);
+ if (0 == diff) {
+ if (valuePartition != null && cq.valuePartition != null) {
+ return valuePartition.compareTo(cq.valuePartition);
+ } else if (valuePartition == null && cq.valuePartition == null) {
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+ }
+ return diff;
+ }
+
+ public int getMaxValueLength() {
+ return this.maxValueLength;
+ }
+
+ public ValueType getType() {
+ return this.type;
+ }
+
+ public enum ValueType {
+ String, Int, Float, Long, Double, Short, Byte, Char
+ };
+
+ // TODO - Include valuePartition also into this
+ public String toString() {
+ return "CF : " + getColumnFamilyString() + ",Qualifier : " + getQualifierString();
+ }
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Constants.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Constants.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/Constants.java (working copy)
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index;
+
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Constants contains all constants used in indexing
+ *
+ */
+public class Constants {
+
+ public static final int DEFAULT_NUM_RETRIES = 10;
+
+ public static final long DEFAULT_PAUSE = 1000;
+
+ public static final int DEFAULT_RETRY_LONGER_MULTIPLIER = 10;
+
+ public static final byte[] IDX_COL_FAMILY = Bytes.toBytes("d");
+
+ public static final byte[] IDX_COL_QUAL = new byte[0];
+
+ public static final String INDEX_TABLE_SUFFIX = "_idx";
+
+ public static final int DEF_MAX_INDEX_NAME_LENGTH = 18;
+
+ public static final String INDEX_SPEC = "INDEX_SPEC";
+
+ public static final String INDEX_BALANCER_DELEGATOR_CLASS =
+ "hbase.index.loadbalancer.delegator.class";
+
+ /**
+ * Use this as a key to specify index details in {@link HTableDescriptor}
+ * @see HTableDescriptor#setValue(byte[], byte[])
+ */
+ public static final byte[] INDEX_SPEC_KEY = Bytes.toBytes(INDEX_SPEC);
+
+ /**
+ * While scan the index(s) to be used can be explicitly passed from client application. Use this
+ * as the name to pass it in attributes
+ * @see Scan#setAttribute(String, byte[])
+ */
+ public static final String INDEX_EXPRESSION = "indexExpression";
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/master/IndexMasterObserver.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/master/IndexMasterObserver.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/master/IndexMasterObserver.java (working copy)
@@ -0,0 +1,720 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.master;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableDescriptors;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.catalog.MetaReader;
+import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.index.Column;
+import org.apache.hadoop.hbase.index.ColumnQualifier;
+import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType;
+import org.apache.hadoop.hbase.index.Constants;
+import org.apache.hadoop.hbase.index.IndexSpecification;
+import org.apache.hadoop.hbase.index.SecIndexLoadBalancer;
+import org.apache.hadoop.hbase.index.TableIndices;
+import org.apache.hadoop.hbase.index.coprocessor.regionserver.IndexRegionSplitPolicy;
+import org.apache.hadoop.hbase.index.manager.IndexManager;
+import org.apache.hadoop.hbase.index.util.IndexUtils;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.master.AssignmentManager;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.LoadBalancer;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.RegionPlan;
+import org.apache.hadoop.hbase.master.RegionState;
+import org.apache.hadoop.hbase.master.RegionStates;
+import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
+import org.apache.hadoop.hbase.master.handler.DeleteTableHandler;
+import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
+import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZKTable;
+
+/**
+ *
+ * Defines of coprocessor hooks(to support secondary indexing) of operations on
+ * {@link org.apache.hadoop.hbase.master.HMaster} process.
+ *
+ */
+
+public class IndexMasterObserver extends BaseMasterObserver {
+
+ private static final Log LOG = LogFactory.getLog(IndexMasterObserver.class.getName());
+
+ IndexManager idxManager = IndexManager.getInstance();
+
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.hbase.coprocessor.BaseMasterObserver#preCreateTable(org
+ * .apache.hadoop.hbase.coprocessor.ObserverContext, org.apache.hadoop.hbase.HTableDescriptor,
+ * org.apache.hadoop.hbase.HRegionInfo[])
+ */
+ @Override
+ public void preCreateTable(ObserverContext ctx,
+ HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+ LOG.info("Entered into preCreateTable.");
+ MasterServices master = ctx.getEnvironment().getMasterServices();
+ byte[] indexBytes = desc.getValue(Constants.INDEX_SPEC_KEY);
+ if (indexBytes != null) {
+ Map> indexColDetails =
+ new HashMap>();
+ TableName tableName = desc.getTableName();
+ checkEndsWithIndexSuffix(tableName);
+ TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName));
+ TableIndices tableIndices = new TableIndices();
+ tableIndices.readFields(indexBytes);
+ List indices = tableIndices.getIndices();
+ // Even if indices list is empty,it will create index table also.
+ if (indices.isEmpty()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Empty indices. Index table may not created"
+ + " if master goes down in between user table creation");
+ }
+ }
+ LOG.trace("Checking whether column families in "
+ + "index specification are in actual table column familes.");
+ for (IndexSpecification iSpec : indices) {
+ checkColumnsForValidityAndConsistency(desc, iSpec, indexColDetails);
+ }
+ LOG.trace("Column families in index specifications " + "are in actual table column familes.");
+
+ boolean isTableExists =
+ MetaReader.tableExists(master.getCatalogTracker(), desc.getTableName());
+ boolean isIndexTableExists =
+ MetaReader.tableExists(master.getCatalogTracker(), indexTableName);
+
+ if (isTableExists && isIndexTableExists) {
+ throw new TableExistsException(desc.getTableName());
+ } else if (isIndexTableExists) {
+ disableAndDeleteTable(master, indexTableName);
+ }
+ idxManager.addIndexForTable(desc.getNameAsString(), indices);
+ }
+ LOG.info("Exiting from preCreateTable.");
+ }
+
+ @Override
+ public void postModifyTableHandler(ObserverContext ctx,
+ TableName tableName, HTableDescriptor htd) throws IOException {
+ String table = tableName.getNameAsString();
+ MasterServices master = ctx.getEnvironment().getMasterServices();
+ List> tableRegionsAndLocations = null;
+ LOG.info("Entering postModifyTable for the table " + table);
+ byte[] indexBytes = htd.getValue(Constants.INDEX_SPEC_KEY);
+ if (indexBytes != null) {
+ TableDescriptors tableDescriptors = master.getTableDescriptors();
+ Map allTableDesc = tableDescriptors.getAll();
+ String indexTableName = IndexUtils.getIndexTableName(table);
+ if (allTableDesc.containsKey(indexTableName)) {
+ // Do table modification
+ TableIndices tableIndices = new TableIndices();
+ tableIndices.readFields(indexBytes);
+ List indices = tableIndices.getIndices();
+ if (indices.isEmpty()) {
+ LOG.error("Empty indices are passed to modify the table " + table);
+ return;
+ }
+ IndexManager idxManager = IndexManager.getInstance();
+ idxManager.removeIndices(table);
+ idxManager.addIndexForTable(table, indices);
+ LOG.info("Successfully updated the indexes for the table " + table + " to " + indices);
+ } else {
+ try {
+ tableRegionsAndLocations =
+ MetaReader.getTableRegionsAndLocations(master.getCatalogTracker(), tableName, true);
+ } catch (InterruptedException e) {
+ LOG.error("Exception while trying to create index table for the existing table " + table);
+ return;
+ }
+ if (tableRegionsAndLocations != null) {
+ HRegionInfo[] regionInfo = new HRegionInfo[tableRegionsAndLocations.size()];
+ for (int i = 0; i < tableRegionsAndLocations.size(); i++) {
+ regionInfo[i] = tableRegionsAndLocations.get(i).getFirst();
+ }
+
+ byte[][] splitKeys = getSplitKeys(regionInfo);
+ createSecondaryIndexTable(htd, splitKeys, master, true);
+ }
+ }
+ }
+ LOG.info("Exiting postModifyTable for the table " + table);
+ }
+
+ private void checkColumnsForValidityAndConsistency(HTableDescriptor desc,
+ IndexSpecification iSpec, Map> indexColDetails)
+ throws IOException {
+ Set cqList = iSpec.getIndexColumns();
+ if (cqList.isEmpty()) {
+ String message =
+ " Index " + iSpec.getName()
+ + " doesn't contain any columns. Each index should contain atleast one column.";
+ LOG.error(message);
+ throw new DoNotRetryIOException(new IllegalArgumentException(message));
+ }
+ for (ColumnQualifier cq : cqList) {
+ if (null == desc.getFamily(cq.getColumnFamily())) {
+ String message =
+ "Column family " + cq.getColumnFamilyString() + " in index specification "
+ + iSpec.getName() + " not in Column families of table " + desc.getNameAsString()
+ + '.';
+ LOG.error(message);
+ throw new DoNotRetryIOException(new IllegalArgumentException(message));
+ }
+ Column column = new Column(cq.getColumnFamily(), cq.getQualifier(), cq.getValuePartition());
+ ValueType type = cq.getType();
+ int maxlength = cq.getMaxValueLength();
+ Pair colDetail = indexColDetails.get(column);
+ if (null != colDetail) {
+ if (!colDetail.getFirst().equals(type) || colDetail.getSecond() != maxlength) {
+ throw new DoNotRetryIOException(new IllegalArgumentException(
+ "ValueType/max value length of column " + column
+ + " not consistent across the indices"));
+ }
+ } else {
+ indexColDetails.put(column, new Pair(type, maxlength));
+ }
+ }
+ }
+
+ private void checkEndsWithIndexSuffix(TableName tableName) throws IOException {
+ if (IndexUtils.isIndexTable(tableName)) {
+ String message =
+ "User table name should not be ends with " + Constants.INDEX_TABLE_SUFFIX + '.';
+ LOG.error(message);
+ throw new DoNotRetryIOException(new IllegalArgumentException(message));
+ }
+ }
+
+ private void disableAndDeleteTable(MasterServices master, TableName tableName) throws IOException {
+ LOG.error(tableName + " already exists. Disabling and deleting table " + tableName + '.');
+ boolean disabled = master.getAssignmentManager().getZKTable().isDisabledTable(tableName);
+ if (false == disabled) {
+ LOG.info("Disabling table " + tableName + '.');
+ new DisableTableHandler(master, tableName, master.getCatalogTracker(),
+ master.getAssignmentManager(), master.getTableLockManager(), false).prepare().process();
+ if (false == master.getAssignmentManager().getZKTable().isDisabledTable(tableName)) {
+ throw new DoNotRetryIOException("Table " + tableName + " not disabled.");
+ }
+ }
+ LOG.info("Disabled table " + tableName + '.');
+ LOG.info("Deleting table " + tableName + '.');
+ new DeleteTableHandler(tableName, master, master).prepare().process();
+ if (true == MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
+ throw new DoNotRetryIOException("Table " + tableName + " not deleted.");
+ }
+ LOG.info("Deleted table " + tableName + '.');
+ }
+
+ @Override
+ public void preCreateTableHandler(ObserverContext ctx,
+ HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+ if (desc.getValue(Constants.INDEX_SPEC_KEY) != null) {
+ LoadBalancer balancer =
+ ctx.getEnvironment().getMasterServices().getAssignmentManager().getBalancer();
+ if (balancer instanceof SecIndexLoadBalancer) {
+ ((SecIndexLoadBalancer) balancer).addIndexedTable(desc.getTableName());
+ }
+ }
+ }
+
+ @Override
+ public void postCreateTableHandler(ObserverContext ctx,
+ HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+ LOG.info("Entered into postCreateTableHandler of table " + desc.getNameAsString() + '.');
+ if (idxManager.getIndicesForTable(desc.getNameAsString()) != null) {
+ MasterServices master = ctx.getEnvironment().getMasterServices();
+ byte[][] splitKeys = getSplitKeys(regions);
+ // In case of post call for the index table creation, it wont be
+ // IndexedHTableDescriptor
+ createSecondaryIndexTable(desc, splitKeys, master, false);
+ // if there is any user scenarios
+ // we can add index datails to index manager
+ }
+ LOG.info("Exiting from postCreateTableHandler of table " + desc.getNameAsString() + '.');
+ }
+
+ /**
+ * @param HTableDescriptor desc
+ * @param HRegionInfo [] regions
+ * @param MasterServices master
+ * @throws NotAllMetaRegionsOnlineException
+ * @throws IOException
+ */
+ private void createSecondaryIndexTable(HTableDescriptor desc, byte[][] splitKeys,
+ MasterServices master, boolean disableTable) throws NotAllMetaRegionsOnlineException,
+ IOException {
+ TableName indexTableName =
+ TableName.valueOf(IndexUtils.getIndexTableName(desc.getNameAsString()));
+ LOG.info("Creating secondary index table " + indexTableName + " for table "
+ + desc.getNameAsString() + '.');
+ HTableDescriptor indexTableDesc = new HTableDescriptor(indexTableName);
+ HColumnDescriptor columnDescriptor = new HColumnDescriptor(Constants.IDX_COL_FAMILY);
+ String dataBlockEncodingAlgo =
+ master.getConfiguration().get("index.data.block.encoding.algo", "NONE");
+ DataBlockEncoding[] values = DataBlockEncoding.values();
+
+ for (DataBlockEncoding dataBlockEncoding : values) {
+ if (dataBlockEncoding.toString().equals(dataBlockEncodingAlgo)) {
+ columnDescriptor.setDataBlockEncoding(dataBlockEncoding);
+ }
+ }
+
+ // TODO read this data from a config file??
+ columnDescriptor.setBlocksize(8 * 1024);// 8KB
+
+ indexTableDesc.addFamily(columnDescriptor);
+ indexTableDesc.setValue(HTableDescriptor.SPLIT_POLICY, IndexRegionSplitPolicy.class.getName());
+ LOG.info("Setting the split policy for the Index Table " + indexTableName + " as "
+ + IndexRegionSplitPolicy.class.getName() + '.');
+ HRegionInfo[] newRegions = getHRegionInfos(indexTableDesc, splitKeys);
+ CreateTableHandler tableHandler =
+ new CreateTableHandler(master, master.getMasterFileSystem(), indexTableDesc,
+ master.getConfiguration(), newRegions, master);
+ tableHandler.prepare();
+ tableHandler.process();
+ // Disable the index table so that when we enable the main table both can be enabled
+ if (disableTable) {
+ new DisableTableHandler(master, indexTableName, master.getCatalogTracker(),
+ master.getAssignmentManager(), master.getTableLockManager(), false).prepare().process();
+ }
+ LOG.info("Created secondary index table " + indexTableName + " for table "
+ + desc.getNameAsString() + '.');
+ }
+
+ private byte[][] getSplitKeys(HRegionInfo[] regions) {
+ byte[][] splitKeys = null;
+ if (null != regions && regions.length > 1) {
+ // for the 1st region always the start key will be empty. We no need to
+ // pass this as a start key item for the index table because this will
+ // be added by HBase any way. So if we pass empty, HBase will create one
+ // extra region with start and end key as empty byte[].
+ splitKeys = new byte[regions.length - 1][];
+ int i = 0;
+ for (HRegionInfo region : regions) {
+ byte[] startKey = region.getStartKey();
+ if (startKey.length > 0) {
+ splitKeys[i++] = startKey;
+ }
+ }
+ }
+ return splitKeys;
+ }
+
+ private HRegionInfo[] getHRegionInfos(HTableDescriptor hTableDescriptor, byte[][] splitKeys) {
+ HRegionInfo[] hRegionInfos = null;
+ if (splitKeys == null || splitKeys.length == 0) {
+ hRegionInfos = new HRegionInfo[] { new HRegionInfo(hTableDescriptor.getTableName()) };
+ } else {
+ int numRegions = splitKeys.length + 1;
+ hRegionInfos = new HRegionInfo[numRegions];
+ byte[] startKey = null;
+ byte[] endKey = null;
+ for (int i = 0; i < numRegions; i++) {
+ endKey = (i == splitKeys.length) ? null : splitKeys[i];
+ hRegionInfos[i] = new HRegionInfo(hTableDescriptor.getTableName(), startKey, endKey);
+ startKey = endKey;
+ }
+ }
+ return hRegionInfos;
+ }
+
+ @Override
+ public void preAssign(ObserverContext ctx, HRegionInfo hri)
+ throws IOException {
+ boolean isRegionInTransition = checkRegionInTransition(ctx, hri);
+ if (isRegionInTransition) {
+ LOG.info("Not calling assign for region " + hri.getRegionNameAsString()
+ + "because the region is already in transition.");
+ ctx.bypass();
+ return;
+ }
+ }
+
+ @Override
+ public void postAssign(ObserverContext ctx, HRegionInfo regionInfo)
+ throws IOException {
+ LOG.info("Entering into postAssign of region " + regionInfo.getRegionNameAsString() + '.');
+
+ if (!IndexUtils.isIndexTable(regionInfo.getTable().getName())) {
+ MasterServices master = ctx.getEnvironment().getMasterServices();
+ LoadBalancer balancer = master.getAssignmentManager().getBalancer();
+ AssignmentManager am = master.getAssignmentManager();
+ RegionStates regionStates = am.getRegionStates();
+ // waiting until user region is removed from transition.
+ long timeout =
+ master.getConfiguration()
+ .getLong("hbase.bulk.assignment.waiton.empty.rit", 5 * 60 * 1000);
+ try {
+ am.waitOnRegionToClearRegionsInTransition(regionInfo, timeout);
+ } catch (InterruptedException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Interrupted while region in assignment.");
+ }
+ }
+ ServerName sn = regionStates.getRegionServerOfRegion(regionInfo);
+ TableName indexTableName =
+ TableName.valueOf(IndexUtils.getIndexTableName(regionInfo.getTableName()));
+ List tableRegions = regionStates.getRegionsOfTable(indexTableName);
+ for (HRegionInfo hRegionInfo : tableRegions) {
+ if (0 == Bytes.compareTo(hRegionInfo.getStartKey(), regionInfo.getStartKey())) {
+ am.addPlan(hRegionInfo.getEncodedName(), new RegionPlan(hRegionInfo, null, sn));
+ LOG.info("Assigning region " + hRegionInfo.getRegionNameAsString() + " to server " + sn
+ + '.');
+ balancer.updateRegionLocation(hRegionInfo, sn);
+ am.assign(hRegionInfo, true, false);
+ break;
+ }
+ }
+ }
+ LOG.info("Exiting from postAssign " + regionInfo.getRegionNameAsString() + '.');
+ }
+
+ @Override
+ public void preUnassign(ObserverContext ctx, HRegionInfo hri,
+ boolean force) throws IOException {
+ boolean isRegionInTransition = checkRegionInTransition(ctx, hri);
+ if (isRegionInTransition) {
+ LOG.info("Not calling move for region because region" + hri.getRegionNameAsString()
+ + " is already in transition.");
+ ctx.bypass();
+ return;
+ }
+ }
+
+ @Override
+ public void preMove(ObserverContext ctx, HRegionInfo hri,
+ ServerName srcServer, ServerName destServer) throws IOException {
+ boolean isRegionInTransition = checkRegionInTransition(ctx, hri);
+ if (isRegionInTransition) {
+ LOG.info("Not calling move for region " + hri.getRegionNameAsString()
+ + "because the region is already in transition.");
+ ctx.bypass();
+ return;
+ }
+ }
+
+ // This is just an additional precaution. This cannot ensure 100% that the RIT regions
+ // will not be picked up.
+ // Because the RIT map that is taken here is the copy of original RIT map and there is
+ // no sync mechanism also.
+ private boolean checkRegionInTransition(ObserverContext ctx,
+ HRegionInfo hri) {
+ MasterServices master = ctx.getEnvironment().getMasterServices();
+ RegionStates regionStates = master.getAssignmentManager().getRegionStates();
+ String tableName = hri.getTable().getNameAsString();
+ if (!IndexUtils.isIndexTable(tableName)) {
+ if (regionStates.isRegionInTransition(hri)) {
+ return true;
+ } else {
+ String indexTableName = IndexUtils.getIndexTableName(tableName);
+ for (Entry region : regionStates.getRegionsInTransition().entrySet()) {
+ HRegionInfo regionInfo = region.getValue().getRegion();
+ if (indexTableName.equals(regionInfo.getTable().getNameAsString())) {
+ if (Bytes.compareTo(hri.getStartKey(), regionInfo.getStartKey()) == 0) {
+ return true;
+ }
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void postMove(ObserverContext ctx, HRegionInfo regionInfo,
+ ServerName srcServer, ServerName destServer) throws IOException {
+ LOG.info("Entering into postMove " + regionInfo.getRegionNameAsString() + '.');
+ if (!IndexUtils.isIndexTable(regionInfo.getTable().getNameAsString())) {
+ MasterServices master = ctx.getEnvironment().getMasterServices();
+ AssignmentManager am = master.getAssignmentManager();
+ // waiting until user region is removed from transition.
+ long timeout =
+ master.getConfiguration()
+ .getLong("hbase.bulk.assignment.waiton.empty.rit", 5 * 60 * 1000);
+ try {
+ am.waitOnRegionToClearRegionsInTransition(regionInfo, timeout);
+ destServer = am.getRegionStates().getRegionServerOfRegion(regionInfo);
+ am.getBalancer().updateRegionLocation(regionInfo, destServer);
+ } catch (InterruptedException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Interrupted while region in assignment.");
+ }
+ }
+ TableName indexTableName =
+ TableName.valueOf(IndexUtils.getIndexTableName(regionInfo.getTable().getNameAsString()));
+ List tableRegions = am.getRegionStates().getRegionsOfTable(indexTableName);
+ for (HRegionInfo indexRegionInfo : tableRegions) {
+ if (0 == Bytes.compareTo(indexRegionInfo.getStartKey(), regionInfo.getStartKey())) {
+ LOG.info("Assigning region " + indexRegionInfo.getRegionNameAsString() + "from "
+ + srcServer + " to server " + destServer + '.');
+ am.getBalancer().updateRegionLocation(indexRegionInfo, destServer);
+ am.addPlan(indexRegionInfo.getEncodedName(), new RegionPlan(indexRegionInfo, null,
+ destServer));
+ am.unassign(indexRegionInfo);
+ }
+ }
+ }
+ LOG.info("Exiting from postMove " + regionInfo.getRegionNameAsString() + '.');
+ }
+
+ @Override
+ public void postDisableTableHandler(ObserverContext ctx,
+ TableName tableName) throws IOException {
+ LOG.info("Entered into postDisableTableHandler of table " + tableName);
+ MasterServices master = ctx.getEnvironment().getMasterServices();
+ AssignmentManager am = master.getAssignmentManager();
+ try {
+ if (!IndexUtils.isIndexTable(tableName.getNameAsString())) {
+ TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName));
+ // Index table may not present following three cases.
+ // 1) Index details are not specified during table creation then index table wont be
+ // created.
+ // 2) Even we specify index details if master restarted in the middle of user table creation
+ // corresponding index table wont be created. But without creating index table user table
+ // wont
+ // be disabled. No need to call disable for index table at that time.
+ // 3) Index table may be deleted but this wont happen without deleting user table.
+ if (am.getZKTable().isTablePresent(indexTableName)) {
+ long timeout =
+ master.getConfiguration().getLong("hbase.bulk.assignment.waiton.empty.rit",
+ 5 * 60 * 1000);
+ // Both user table and index table should not be in enabling/disabling state at a time.
+ // If disable is progress for user table then index table should be in ENABLED state.
+ // If enable is progress for index table wait until table enabled.
+ if (waitUntilTableEnabled(timeout, indexTableName, am.getZKTable())) {
+ new DisableTableHandler(master, indexTableName, master.getCatalogTracker(), am,
+ master.getTableLockManager(), false).process();
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Table " + indexTableName + " not in ENABLED state to disable.");
+ }
+ }
+ }
+ }
+ } finally {
+ // clear user table region plans in secondary index load balancer.
+ clearRegionPlans((HMaster) master, tableName.getNamespaceAsString());
+ }
+ LOG.info("Exiting from postDisableTableHandler of table " + tableName);
+ }
+
+ private void clearRegionPlans(HMaster master, String tableName) {
+ AssignmentManager am = master.getAssignmentManager();
+ if (am.getBalancer() instanceof SecIndexLoadBalancer) {
+ ((SecIndexLoadBalancer) am.getBalancer()).clearTableRegionPlans(tableName);
+ }
+ }
+
+ @Override
+ public void postEnableTableHandler(ObserverContext ctx,
+ TableName tableName) throws IOException {
+ LOG.info("Entered into postEnableTableHandler of table " + tableName);
+ if (!IndexUtils.isIndexTable(tableName.getNameAsString())) {
+ MasterServices master = ctx.getEnvironment().getMasterServices();
+ AssignmentManager am = master.getAssignmentManager();
+ TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName));
+ // Index table may not present in three cases
+ // 1) Index details are not specified during table creation then index table wont be created.
+ // 2) Even we specify index details if master restarted in the middle of user table creation
+ // corresponding index table wont be created. Then no need to call enable for index table
+ // because it will be created as part of preMasterInitialization and enable.
+ // 3) Index table may be deleted but this wont happen without deleting user table.
+ if (true == am.getZKTable().isTablePresent(indexTableName)) {
+ long timeout =
+ master.getConfiguration().getLong("hbase.bulk.assignment.waiton.empty.rit",
+ 5 * 60 * 1000);
+ // Both user table and index table should not be in enabling/disabling state at a time.
+ // If enable is progress for user table then index table should be in disabled state.
+ // If disable is progress for index table wait until table disabled.
+ if (waitUntilTableDisabled(timeout, indexTableName, am.getZKTable())) {
+ new EnableTableHandler(master, indexTableName, master.getCatalogTracker(), am,
+ master.getTableLockManager(), false).prepare().process();
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Table " + indexTableName + " not in DISABLED state to enable.");
+ }
+ }
+ }
+ }
+ LOG.info("Exiting from postEnableTableHandler of table " + tableName);
+
+ }
+
+ private boolean waitUntilTableDisabled(long timeout, TableName tableName, ZKTable zk) {
+ long startTime = System.currentTimeMillis();
+ long remaining = timeout;
+ boolean disabled = false;
+ while (!(disabled = zk.isDisabledTable(tableName)) && remaining > 0) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Interrupted while waiting for table" + tableName + " set to DISABLED.");
+ }
+ }
+ remaining = timeout - (System.currentTimeMillis() - startTime);
+ }
+ if (remaining <= 0) {
+ return disabled;
+ } else {
+ return true;
+ }
+ }
+
+ private boolean waitUntilTableEnabled(long timeout, TableName tableName, ZKTable zk) {
+ long startTime = System.currentTimeMillis();
+ long remaining = timeout;
+ boolean enabled = false;
+ while (!(enabled = zk.isEnabledTable(tableName)) && remaining > 0) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Interrupted while waiting for table " + tableName + "state set to ENABLED.");
+ }
+ }
+ remaining = timeout - (System.currentTimeMillis() - startTime);
+ }
+ if (remaining <= 0) {
+ return enabled;
+ } else {
+ return true;
+ }
+ }
+
+ @Override
+ public void postDeleteTableHandler(ObserverContext ctx,
+ TableName tableName) throws IOException {
+ LOG.info("Entered into postDeleteTableHandler of table " + tableName + '.');
+ MasterServices master = ctx.getEnvironment().getMasterServices();
+ TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName));
+ boolean indexTablePresent =
+ master.getAssignmentManager().getZKTable().isTablePresent(indexTableName);
+ // Not checking for disabled state because before deleting user table both user and index table
+ // should be disabled.
+ if ((!IndexUtils.isIndexTable(tableName)) && indexTablePresent) {
+ LoadBalancer balancer = master.getAssignmentManager().getBalancer();
+ if (balancer instanceof SecIndexLoadBalancer) {
+ ((SecIndexLoadBalancer) balancer).removeIndexedTable(tableName);
+ }
+ DeleteTableHandler dth = new DeleteTableHandler(indexTableName, master, master);
+ dth.prepare();
+ dth.process();
+ }
+ LOG.info("Exiting from postDeleteTableHandler of table " + tableName + '.');
+ }
+
+ @Override
+ public void preMasterInitialization(ObserverContext ctx)
+ throws IOException {
+ LOG.info("Entering into preMasterInitialization.");
+ MasterServices master = ctx.getEnvironment().getMasterServices();
+ AssignmentManager am = master.getAssignmentManager();
+ ZKTable zkTable = am.getZKTable();
+ long timeout =
+ master.getConfiguration().getLong("hbase.bulk.assignment.waiton.empty.rit", 5 * 60 * 1000);
+ try {
+ am.waitUntilNoRegionsInTransition(timeout);
+ } catch (InterruptedException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Interrupted while waiting for the regions in transition to complete.", e);
+ }
+ }
+
+ TableDescriptors tableDescriptors = master.getTableDescriptors();
+ Map descMap = tableDescriptors.getAll();
+ Collection htds = descMap.values();
+ for (HTableDescriptor htd : htds) {
+ if (!IndexUtils.isIndexTable(htd.getNameAsString()) && !htd.isMetaRegion()
+ && !htd.getTableName().isSystemTable()) {
+ if (htd.getValue(Constants.INDEX_SPEC_KEY) != null) {
+ TableName tableName = htd.getTableName();
+ LoadBalancer balancer = master.getAssignmentManager().getBalancer();
+ if (balancer instanceof SecIndexLoadBalancer) {
+ ((SecIndexLoadBalancer) balancer).addIndexedTable(tableName);
+ }
+ TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName));
+ // TODO: check table exists from table descriptors list.
+ boolean tableExists = MetaReader.tableExists(master.getCatalogTracker(), tableName);
+ boolean indexTableExists =
+ MetaReader.tableExists(master.getCatalogTracker(), indexTableName);
+ if (tableExists && !indexTableExists) {
+ LOG.info("Table has index specification details but " + "no corresponding index table.");
+ List regions =
+ MetaReader.getTableRegions(master.getCatalogTracker(), tableName);
+ HRegionInfo[] regionsArray = new HRegionInfo[regions.size()];
+ byte[][] splitKeys = getSplitKeys(regions.toArray(regionsArray));
+ createSecondaryIndexTable(htd, splitKeys, master, zkTable.isDisabledTable(tableName));
+ } else if (true == tableExists && true == indexTableExists) {
+ // If both tables are present both should be in same state in zookeeper. If tables are
+ // partially enabled or disabled they will be processed as part of recovery
+ // enabling/disabling tables.
+ // if user table is in ENABLED state and index table is in DISABLED state means master
+ // restarted as soon as user table enabled. So here we need to enable index table.
+ if (zkTable.isEnabledTable(tableName) && zkTable.isDisabledTable(indexTableName)) {
+ new EnableTableHandler(master, indexTableName, master.getCatalogTracker(), am,
+ master.getTableLockManager(), false).prepare().process();
+ } else if (zkTable.isDisabledTable(tableName) && zkTable.isEnabledTable(indexTableName)) {
+ // If user table is in DISABLED state and index table is in ENABLED state means master
+ // restarted as soon as user table disabled. So here we need to disable index table.
+ new DisableTableHandler(master, indexTableName, master.getCatalogTracker(), am,
+ master.getTableLockManager(), false).prepare().process();
+ // clear index table region plans in secondary index load balancer.
+ clearRegionPlans((HMaster) master, indexTableName.getNameAsString());
+ }
+ }
+ }
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Balancing after master initialization.");
+ }
+
+ try {
+ master.getAssignmentManager().waitUntilNoRegionsInTransition(timeout);
+ } catch (InterruptedException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Interrupted while waiting for the regions in transition to complete.", e);
+ }
+ }
+ ((HMaster) master).balanceInternals();
+ LOG.info("Exiting from preMasterInitialization.");
+ }
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/BackwardSeekableRegionScanner.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/BackwardSeekableRegionScanner.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/BackwardSeekableRegionScanner.java (working copy)
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+
+public class BackwardSeekableRegionScanner implements SeekAndReadRegionScanner {
+
+ private ReInitializableRegionScanner delegator;
+
+ private Scan scan;
+
+ private HRegion hRegion;
+
+ private byte[] startRow;
+
+ private boolean closed = false;
+
+ public BackwardSeekableRegionScanner(ReInitializableRegionScanner delegator, Scan scan,
+ HRegion hRegion, byte[] startRow) {
+ this.delegator = delegator;
+ this.scan = scan;
+ this.hRegion = hRegion;
+ this.startRow = startRow;
+ }
+
+ Scan getScan() {
+ return scan;
+ }
+
+ byte[] getStartRow() {
+ return startRow;
+ }
+
+ // For testing.
+ RegionScanner getDelegator() {
+ return delegator;
+ }
+
+ @Override
+ public HRegionInfo getRegionInfo() {
+ return this.delegator.getRegionInfo();
+ }
+
+ @Override
+ public boolean isFilterDone() throws IOException {
+ return this.delegator.isFilterDone();
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ this.delegator.close();
+ closed = true;
+ }
+
+ @Override
+ public boolean isClosed() {
+ return closed;
+ }
+
+ @Override
+ public synchronized boolean next(List results) throws IOException {
+ return next(results, this.scan.getBatch());
+ }
+
+ @Override
+ public boolean next(List result, int limit) throws IOException {
+ boolean hasNext = false;
+ try {
+ if (this.delegator.isClosed()) return false;
+ hasNext = this.delegator.next(result, limit);
+ } catch (SeekUnderValueException e) {
+ Scan newScan = new Scan(this.scan);
+ // Start from the point where we got stopped because of seek backward
+ newScan.setStartRow(getLatestSeekpoint());
+ this.delegator.reInit(this.hRegion.getScanner(newScan));
+ hasNext = next(result, limit);
+ }
+ return hasNext;
+ }
+
+ @Override
+ public synchronized boolean reseek(byte[] row) throws IOException {
+ return this.delegator.reseek(row);
+ }
+
+ @Override
+ public void addSeekPoints(List seekPoints) {
+ this.delegator.addSeekPoints(seekPoints);
+ }
+
+ @Override
+ public boolean seekToNextPoint() throws IOException {
+ return this.delegator.seekToNextPoint();
+ }
+
+ @Override
+ public byte[] getLatestSeekpoint() {
+ return this.delegator.getLatestSeekpoint();
+ }
+
+ @Override
+ public long getMaxResultSize() {
+ return this.delegator.getMaxResultSize();
+ }
+
+ @Override
+ public long getMvccReadPoint() {
+ return this.delegator.getMvccReadPoint();
+ }
+
+ @Override
+ public boolean nextRaw(List result) throws IOException {
+ return next(result);
+ }
+
+ @Override
+ public boolean nextRaw(List result, int limit) throws IOException {
+ return next(result, limit);
+ }
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueDetail.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueDetail.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueDetail.java (working copy)
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.index.Column;
+import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType;
+import org.apache.hadoop.hbase.index.ValuePartition;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class FilterColumnValueDetail {
+ protected byte[] cf;
+ protected byte[] qualifier;
+ protected byte[] value;
+ protected CompareOp compareOp;
+ protected Column column;
+ protected int maxValueLength;
+ protected ValueType valueType;
+
+ public FilterColumnValueDetail(byte[] cf, byte[] qualifier, byte[] value, CompareOp compareOp) {
+ this.cf = cf;
+ this.qualifier = qualifier;
+ this.value = value;
+ this.compareOp = compareOp;
+ this.column = new Column(this.cf, this.qualifier);
+ }
+
+ public FilterColumnValueDetail(byte[] cf, byte[] qualifier, byte[] value,
+ ValuePartition valuePartition, CompareOp compareOp) {
+ this.cf = cf;
+ this.qualifier = qualifier;
+ this.value = value;
+ this.compareOp = compareOp;
+ this.column = new Column(this.cf, this.qualifier, valuePartition);
+ }
+
+ public FilterColumnValueDetail(Column column, byte[] value, CompareOp compareOp) {
+ this.cf = column.getFamily();
+ this.qualifier = column.getQualifier();
+ this.value = value;
+ this.compareOp = compareOp;
+ this.column = column;
+ }
+
+ public boolean equals(Object obj) {
+ if (!(obj instanceof FilterColumnValueDetail)) return false;
+ FilterColumnValueDetail that = (FilterColumnValueDetail) obj;
+ if (!this.column.equals(that.column)) {
+ return false;
+ }
+ // Need to check.
+ if (this.value != null && that.value != null) {
+ if (!(Bytes.equals(this.value, that.value))) return false;
+ } else if (this.value == null && that.value == null) {
+ return true;
+ } else {
+ return false;
+ }
+ return true;
+ }
+
+ public int hashCode() {
+ return this.column.hashCode();
+ }
+
+ public String toString() {
+ return String.format("%s (%s, %s, %s, %s, %s)", this.getClass().getSimpleName(),
+ Bytes.toStringBinary(this.cf), Bytes.toStringBinary(this.qualifier), this.valueType.name(),
+ this.compareOp.name(), Bytes.toStringBinary(this.value));
+ }
+
+ public Column getColumn() {
+ return this.column;
+ }
+
+ public byte[] getValue() {
+ return this.value;
+ }
+
+ protected void setValue(byte[] value) {
+ this.value = value;
+ }
+
+}
\ No newline at end of file
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueRange.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueRange.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterColumnValueRange.java (working copy)
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.index.Column;
+import org.apache.hadoop.hbase.index.ValuePartition;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class FilterColumnValueRange extends FilterColumnValueDetail {
+ private CompareOp upperBoundCompareOp;
+ private byte[] upperBoundValue;
+
+ public FilterColumnValueRange(byte[] cf, byte[] qualifier, byte[] lowerBoundValue,
+ CompareOp lowerBoundCompareOp, byte[] upperBoundValue, CompareOp upperBoundCompareOp) {
+ super(cf, qualifier, lowerBoundValue, lowerBoundCompareOp);
+ this.upperBoundCompareOp = upperBoundCompareOp;
+ this.upperBoundValue = upperBoundValue;
+ }
+
+ public FilterColumnValueRange(byte[] cf, byte[] qualifier, ValuePartition vp,
+ byte[] lowerBoundValue, CompareOp lowerBoundCompareOp, byte[] upperBoundValue,
+ CompareOp upperBoundCompareOp) {
+ super(cf, qualifier, lowerBoundValue, vp, lowerBoundCompareOp);
+ this.upperBoundCompareOp = upperBoundCompareOp;
+ this.upperBoundValue = upperBoundValue;
+ }
+
+ public FilterColumnValueRange(Column column, byte[] lowerBoundValue,
+ CompareOp lowerBoundCompareOp, byte[] upperBoundValue, CompareOp upperBoundCompareOp) {
+ super(column, lowerBoundValue, lowerBoundCompareOp);
+ this.upperBoundCompareOp = upperBoundCompareOp;
+ this.upperBoundValue = upperBoundValue;
+ }
+
+ // No need to have the hashCode() and equals(Object obj) implementation here. Super class
+ // implementation checks for the CF name and qualifier name which is sufficient.
+
+ public String toString() {
+ return String.format("%s (%s, %s, %s, %s, %s, %s, %s)", this.getClass().getSimpleName(), Bytes
+ .toStringBinary(this.cf), Bytes.toStringBinary(this.qualifier), this.valueType.name(),
+ this.compareOp.name(), Bytes.toStringBinary(this.value),
+ this.upperBoundCompareOp == null ? "" : this.upperBoundCompareOp.name(),
+ this.upperBoundValue == null ? "" : Bytes.toStringBinary(this.upperBoundValue));
+ }
+
+ public CompareOp getUpperBoundCompareOp() {
+ return this.upperBoundCompareOp;
+ }
+
+ public byte[] getUpperBoundValue() {
+ return this.upperBoundValue;
+ }
+
+ // The equals method is a bit tricky.
+ // Needs one more eye on this
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) return true;
+ FilterColumnValueRange fcvr = null;
+ if (other instanceof FilterColumnValueRange && super.equals(other)) {
+ fcvr = (FilterColumnValueRange) other;
+ if (this.upperBoundValue != null && fcvr.getUpperBoundValue() != null) {
+ if (Bytes.compareTo(this.upperBoundValue, fcvr.getUpperBoundValue()) != 0) {
+ return false;
+ }
+ } else if (this.upperBoundValue == null && fcvr.getUpperBoundValue() == null) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+}
\ No newline at end of file
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterGroupingWorker.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterGroupingWorker.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterGroupingWorker.java (working copy)
@@ -0,0 +1,527 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.index.Column;
+import org.apache.hadoop.hbase.index.ValuePartition;
+import org.apache.hadoop.hbase.index.filter.SingleColumnRangeFilter;
+import org.apache.hadoop.hbase.index.filter.SingleColumnValuePartitionFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * This class does the grouping work for different filterlists.
+ * Like if we i have 3 different filter list and all have MUST_PASS_ALL condition
+ * finally this class groups the filters lists into one filter list with all as MUST_PASS_ALL.
+ * Also it checks if the combination of the filters list is given correctly.
+ * For eg: If i have c1 > 10 and c1 < 5. This is wrong combination.
+ */
+public class FilterGroupingWorker {
+
+ private static final Log LOG = LogFactory.getLog(FilterGroupingWorker.class);
+
+ private Map> colWithOperators =
+ new HashMap>();
+ private Map> colWithOperatorsOfOR = new HashMap>();
+
+ public Filter group(Filter filter) {
+ if (filter instanceof FilterList) {
+ FilterList fList = (FilterList) filter;
+ // We need to create a new FL here taking up only the filters of our interest
+ FilterList newFList = new FilterList(fList.getOperator());
+ List filters = fList.getFilters();
+ if (fList.getOperator() == Operator.MUST_PASS_ONE) {
+ for (Filter subFilter : filters) {
+ Filter resultFilter = handleFilterWithinOR(subFilter);
+ // If result filter is not SingleColumnValueFilter or filter list that means OR branch is
+ // having different type of filter other than SCVF. In that case we should not consider
+ // the OR branch for scanning.
+ if (resultFilter instanceof FilterList) {
+ newFList.addFilter(resultFilter);
+ } else if (resultFilter != null) {
+ // This means OR filter list have at least one filter other than SCVF(may be other
+ // child OR branches).
+ return null;
+ }
+ }
+ addORColsToFinalList(newFList);
+ if (newFList.getFilters().isEmpty()) {
+ return null;
+ }
+ return newFList;
+ } else {
+ // AND condition as long as the condition is AND in one sub tree all those can be
+ // grouped under one AND parent(new one).
+ for (Filter subFilter : filters) {
+ Filter group = handleFilterWithinAND(subFilter);
+ // group is null means, all are AND conditions and will be handled at once with the
+ // below createFinalFilter
+ if (group != null) {
+ newFList.addFilter(group);
+ }
+ }
+ addANDColsToFinalList(newFList);
+ if (newFList.getFilters().isEmpty()) {
+ return null;
+ }
+ return newFList;
+ }
+ } else if (filter instanceof SingleColumnValueFilter
+ || filter instanceof SingleColumnRangeFilter) {
+ return filter;
+ }
+ return null;
+ }
+
+ private Filter handleFilterWithinAND(Filter filter) {
+ if (filter instanceof FilterList) {
+ FilterList fList = (FilterList) filter;
+ if (fList.getOperator() == Operator.MUST_PASS_ONE) {
+ return new FilterGroupingWorker().group(fList);
+ } else {
+ List filters = fList.getFilters();
+ for (Filter subFilter : filters) {
+ handleFilterWithinAND(subFilter);
+ }
+ }
+ } else if (filter instanceof SingleColumnValueFilter) {
+ handleScvf((SingleColumnValueFilter) filter);
+ } // TODO when we expose SingleColumnRangeFilter to handle that also here.
+ return null;
+ }
+
+ /**
+ * Since you can use Filter Lists as children of Filter Lists, you can create a hierarchy of
+ * filters to be evaluated. In the hierarchy if OR branch having any filter type other than SCVF
+ * as child then we should not consider the branch for scanning because we cannot fetch seek
+ * points from other type of filters without column and value details. Ex: AND AND
+ * __________|_______ | | | --> SCVF OR SCVF _______|______ | | ROWFILTER SVCF If the OR is root
+ * then we should skip index table scanning for this filter. OR _______|______ --> null | |
+ * ROWFILTER SVCF If the OR is child of another OR branch then parent OR branch will be excluded
+ * for scanning. Ex: AND AND __________|_______ | | | --> SCVF OR SCVF _______|______ | | OR SVCF
+ * _______|______ | | ROWFILTER SVCF
+ * @param filter
+ * @return if filter is filter list with AND condition then we will return AND branch after
+ * grouping. if filter is filter list with OR condition return null if no children is of
+ * type other than SCVF or filter list else return different filter. if filter is SCVF
+ * then return null. returning null means we are combining the filter(s) with children of
+ * parent OR filter to perform optimizations.
+ */
+ private Filter handleFilterWithinOR(Filter filter) {
+ if (filter instanceof FilterList) {
+ FilterList fList = (FilterList) filter;
+ if (fList.getOperator() == Operator.MUST_PASS_ONE) {
+ List filters = fList.getFilters();
+ Filter resultFilter = null;
+ for (Filter subFilter : filters) {
+ // If this OR branch in the filter list have filter type other than SCVF we should report
+ // it to parent by returning the other type of filter in such a way that the branch will
+ // be skipped from index scan.
+ resultFilter = handleFilterWithinOR(subFilter);
+ if (resultFilter == null || (resultFilter instanceof FilterList)) {
+ continue;
+ } else {
+ return resultFilter;
+ }
+ }
+ return null;
+ } else {
+ return new FilterGroupingWorker().group(fList);
+ }
+ } else if (filter instanceof SingleColumnValueFilter) {
+ handleScvfOfOR((SingleColumnValueFilter) filter);
+ return null;
+ }// TODO when we expose SingleColumnRangeFilter to handle that also here.
+ // filter other than SingleColumnValueFilter.
+ return filter;
+ }
+
+ private void handleScvfOfOR(SingleColumnValueFilter scvf) {
+ ValuePartition vp = null;
+ if (scvf instanceof SingleColumnValuePartitionFilter) {
+ vp = ((SingleColumnValuePartitionFilter) scvf).getValuePartition();
+ }
+ Column key = new Column(scvf.getFamily(), scvf.getQualifier(), vp);
+ if (colWithOperatorsOfOR.get(key) == null) {
+ List valueList = new ArrayList(1);
+ valueList.add(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf));
+ colWithOperatorsOfOR.put(key, valueList);
+ } else {
+ List valueList = colWithOperatorsOfOR.get(key);
+ Iterator valueListItr = valueList.iterator();
+ CompareOp operator = scvf.getOperator();
+ byte[] value = scvf.getComparator().getValue();
+ Value prevValueObj = null;
+ while (valueListItr.hasNext()) {
+ prevValueObj = valueListItr.next();
+ // TODO As Anoop said we may have to check the Value type also..
+ // We can not compare and validate this way. btw "a" and "K".
+ // Only in case of Numeric col type we can have this check.
+ byte[] prevValue = prevValueObj.getValue();
+ int result = Bytes.compareTo(prevValue, value);
+ CompareOp prevOperator = prevValueObj.getOperator();
+ switch (operator) {
+ case GREATER:
+ if (prevOperator == CompareOp.GREATER || prevOperator == CompareOp.GREATER_OR_EQUAL) {
+ if (result > 0) {
+ valueListItr.remove();
+ } else {
+ // Already you found less or equal value than present value means present filter will
+ // return subset of previous filter. No need to add it to list.
+ return;
+ }
+ } else if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) {
+ // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we
+ // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to
+ // list.
+ } else if (prevOperator == CompareOp.EQUAL) {
+ if (result > 0) {
+ valueListItr.remove();
+ } else if (result == 0) {
+ // remove this entry and convert GREATER to GREATER_OR_EQUAL
+ valueListItr.remove();
+ SingleColumnValueFilter newScvf = null;
+ if (vp == null) {
+ newScvf =
+ new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(),
+ CompareOp.GREATER_OR_EQUAL, scvf.getComparator());
+ } else {
+ newScvf =
+ new SingleColumnValuePartitionFilter(scvf.getFamily(), scvf.getQualifier(),
+ CompareOp.GREATER_OR_EQUAL, scvf.getComparator(), vp);
+ }
+ Value newValue = new Value(CompareOp.GREATER_OR_EQUAL, prevValue, newScvf);
+ valueList.add(newValue);
+ return;
+ }
+ }
+ break;
+ case GREATER_OR_EQUAL:
+ if (prevOperator == CompareOp.GREATER || prevOperator == CompareOp.GREATER_OR_EQUAL) {
+ if (result >= 0) {
+ valueListItr.remove();
+ } else {
+ // Already you found less value than present value means present filter will
+ // return subset of previous filter. No need to add it to list.
+ return;
+ }
+ } else if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) {
+ // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we
+ // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to
+ // list.
+ } else if (prevOperator == CompareOp.EQUAL) {
+ if (result >= 0) {
+ valueListItr.remove();
+ }
+ }
+ break;
+ case LESS:
+ if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) {
+ if (result < 0) {
+ valueListItr.remove();
+ } else {
+ // Already you found less or equal value than present value means present filter will
+ // return subset of previous filter. No need to add it to list.
+ return;
+ }
+ } else if (prevOperator == CompareOp.GREATER
+ || prevOperator == CompareOp.GREATER_OR_EQUAL) {
+ // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we
+ // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to
+ // list.
+ } else if (prevOperator == CompareOp.EQUAL) {
+ if (result < 0) {
+ valueListItr.remove();
+ } else if (result == 0) {
+ // remove this entry and convert LESS to LESS_OR_EQUAL
+ valueListItr.remove();
+ SingleColumnValueFilter newScvf = null;
+ if (vp == null) {
+ newScvf =
+ new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(),
+ CompareOp.LESS_OR_EQUAL, scvf.getComparator());
+ } else {
+ newScvf =
+ new SingleColumnValuePartitionFilter(scvf.getFamily(), scvf.getQualifier(),
+ CompareOp.LESS_OR_EQUAL, scvf.getComparator(), vp);
+ }
+ Value newValue = new Value(CompareOp.LESS_OR_EQUAL, prevValue, newScvf);
+ valueList.add(newValue);
+ return;
+ }
+ }
+ break;
+ case LESS_OR_EQUAL:
+ if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) {
+ if (result <= 0) {
+ valueListItr.remove();
+ } else {
+ // Already you found less or equal value than present value means present filter will
+ // return subset of previous filter. No need to add it to list.
+ return;
+ }
+ } else if (prevOperator == CompareOp.GREATER
+ || prevOperator == CompareOp.GREATER_OR_EQUAL) {
+ // Need to handle conditions like previous is c1<5 and current c1>2. In these cases we
+ // can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to
+ // list.
+ } else if (prevOperator == CompareOp.EQUAL) {
+ // If we dont want to do conversion we can add into first if condition.
+ if (result <= 0) {
+ valueListItr.remove();
+ } else if (result == 0) {
+ // remove this entry and convert GREATER to GREATER_OR_EQUAL
+ }
+ }
+ break;
+ case EQUAL:
+ if (prevOperator == CompareOp.GREATER) {
+ if (result < 0) {
+ return;
+ } else if (result == 0) {
+ valueListItr.remove();
+ SingleColumnValueFilter newScvf = null;
+ if (vp == null) {
+ newScvf =
+ new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(),
+ CompareOp.GREATER_OR_EQUAL, scvf.getComparator());
+ } else {
+ newScvf =
+ new SingleColumnValuePartitionFilter(scvf.getFamily(), scvf.getQualifier(),
+ CompareOp.GREATER_OR_EQUAL, scvf.getComparator(), vp);
+ }
+ Value newValue = new Value(CompareOp.GREATER_OR_EQUAL, prevValue, newScvf);
+ valueList.add(newValue);
+ return;
+ }
+ } else if (prevOperator == CompareOp.GREATER_OR_EQUAL) {
+ if (result <= 0) {
+ return;
+ }
+ } else if (prevOperator == CompareOp.LESS) {
+ if (result > 0) {
+ return;
+ } else if (result == 0) {
+ valueListItr.remove();
+ SingleColumnValueFilter newScvf = null;
+ if (vp == null) {
+ newScvf =
+ new SingleColumnValueFilter(scvf.getFamily(), scvf.getQualifier(),
+ CompareOp.LESS_OR_EQUAL, scvf.getComparator());
+ } else {
+ newScvf =
+ new SingleColumnValuePartitionFilter(scvf.getFamily(), scvf.getQualifier(),
+ CompareOp.LESS_OR_EQUAL, scvf.getComparator(), vp);
+ }
+ Value newValue = new Value(CompareOp.LESS_OR_EQUAL, prevValue, newScvf);
+ valueList.add(newValue);
+ return;
+ }
+ } else if (prevOperator == CompareOp.LESS_OR_EQUAL) {
+ if (result >= 0) {
+ return;
+ }
+ } else if (prevOperator == CompareOp.EQUAL) {
+ if (result == 0) {
+ // Already same filter exists with same condiftion.
+ return;
+ }
+ }
+ break;
+ case NOT_EQUAL:
+ case NO_OP:
+ // Need to check this
+ break;
+ }
+ }
+ valueList.add(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf));
+ }
+ }
+
+ private void handleScvf(SingleColumnValueFilter scvf) {
+ ValuePartition vp = null;
+ if (scvf instanceof SingleColumnValuePartitionFilter) {
+ vp = ((SingleColumnValuePartitionFilter) scvf).getValuePartition();
+ }
+ Column column = new Column(scvf.getFamily(), scvf.getQualifier(), vp);
+ Pair pair = colWithOperators.get(column);
+ if (pair == null) {
+ pair = new Pair();
+ // The first operator should be set here
+ pair.setFirst(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf));
+ colWithOperators.put(column, pair);
+ } else {
+ if (pair.getFirst() != null && pair.getSecond() == null) {
+ // TODO As Anoop said we may have to check the Value type also..
+ // We can not compare and validate this way. btw "a" and "K".
+ // Only in case of Numeric col type we can have this check.
+ byte[] curBoundValue = scvf.getComparator().getValue();
+ byte[] prevBoundValue = pair.getFirst().getValue();
+ int result = Bytes.compareTo(prevBoundValue, curBoundValue);
+ CompareOp curBoundOperator = scvf.getOperator();
+ CompareOp prevBoundOperator = pair.getFirst().getOperator();
+ switch (curBoundOperator) {
+ case GREATER:
+ case GREATER_OR_EQUAL:
+ if (prevBoundOperator == CompareOp.GREATER
+ || prevBoundOperator == CompareOp.GREATER_OR_EQUAL) {
+ LOG.warn("Wrong usage. It should be < > || > <. Cannot be > >");
+ if (result > 1) {
+ pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf));
+ }
+ pair.setSecond(null);
+ } else if (prevBoundOperator == CompareOp.LESS
+ || prevBoundOperator == CompareOp.LESS_OR_EQUAL) {
+ if (result < 1) {
+ LOG.warn("Possible wrong usage as there cannot be a value < 10 and > 20");
+ pair.setFirst(null);
+ pair.setSecond(null);
+ } else {
+ pair.setSecond(new Value(curBoundOperator, curBoundValue, scvf));
+ }
+ } else if (prevBoundOperator == CompareOp.EQUAL) {
+ LOG.warn("Use the equal operator and ignore the current one");
+ pair.setSecond(null);
+ }
+ break;
+ case LESS:
+ case LESS_OR_EQUAL:
+ if (prevBoundOperator == CompareOp.LESS || prevBoundOperator == CompareOp.LESS_OR_EQUAL) {
+ LOG.warn("Wrong usage. It should be < > || > <. Cannot be > >");
+ if (result < 1) {
+ pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf));
+ }
+ pair.setSecond(null);
+ } else if (prevBoundOperator == CompareOp.GREATER
+ || prevBoundOperator == CompareOp.GREATER_OR_EQUAL) {
+ if (result > 1) {
+ LOG.warn("Possible wrong usage as there cannot be a value < 10 and > 20");
+ pair.setFirst(null);
+ pair.setSecond(null);
+ } else {
+ pair.setSecond(new Value(curBoundOperator, curBoundValue, scvf));
+ }
+ } else if (prevBoundOperator == CompareOp.EQUAL) {
+ LOG.warn("Use the EQUAL operator only and ignore the current one.");
+ pair.setSecond(null);
+ }
+ break;
+ case EQUAL:
+ // For equal condition give priority to equals only..
+ // If the prevOperator is also == and the current is also ==
+ // take the second one.(Currently)
+ if (prevBoundOperator == CompareOp.LESS || prevBoundOperator == CompareOp.LESS_OR_EQUAL
+ || prevBoundOperator == CompareOp.EQUAL || prevBoundOperator == CompareOp.GREATER
+ || prevBoundOperator == CompareOp.GREATER_OR_EQUAL) {
+ pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf));
+ pair.setSecond(null);
+ }
+ break;
+ case NOT_EQUAL:
+ case NO_OP:
+ // Need to check this
+ break;
+ }
+ } else {
+ LOG.warn("Am getting an extra comparison coming for the same col family."
+ + "I cannot have 3 conditions on the same column");
+ pair.setFirst(null);
+ pair.setSecond(null);
+ }
+ }
+ }
+
+ private void addANDColsToFinalList(FilterList filterList) {
+ for (Entry> entry : colWithOperators.entrySet()) {
+ Pair value = entry.getValue();
+ if (value.getFirst() != null && value.getSecond() != null) {
+ // Here we are introducing a new Filter
+ SingleColumnRangeFilter rangeFltr =
+ new SingleColumnRangeFilter(entry.getKey().getFamily(), entry.getKey().getQualifier(),
+ entry.getKey().getValuePartition(), value.getFirst().getValue(), value.getFirst()
+ .getOperator(), value.getSecond().getValue(), value.getSecond().getOperator());
+ filterList.addFilter(rangeFltr);
+ } else if (value.getFirst() != null) {
+ if (value.getFirst().getOperator() == CompareOp.EQUAL) {
+ filterList.addFilter(value.getFirst().getFilter());
+ } else {
+ SingleColumnRangeFilter rangeFltr =
+ new SingleColumnRangeFilter(entry.getKey().getFamily(),
+ entry.getKey().getQualifier(), entry.getKey().getValuePartition(), value
+ .getFirst().getValue(), value.getFirst().getOperator(), null, null);
+ filterList.addFilter(rangeFltr);
+ }
+ }
+ }
+ }
+
+ private void addORColsToFinalList(FilterList filterList) {
+ for (Entry> entry : colWithOperatorsOfOR.entrySet()) {
+ List valueList = entry.getValue();
+ for (Value value : valueList) {
+ if (value.getOperator() == CompareOp.EQUAL) {
+ filterList.addFilter(value.getFilter());
+ } else {
+ SingleColumnRangeFilter rangeFltr =
+ new SingleColumnRangeFilter(entry.getKey().getFamily(),
+ entry.getKey().getQualifier(), entry.getKey().getValuePartition(),
+ value.getValue(), value.getOperator(), null, null);
+ filterList.addFilter(rangeFltr);
+ }
+ }
+ }
+ }
+
+ private static class Value {
+ private CompareOp operator;
+ private byte[] value;
+ private Filter filter;
+
+ public Value(CompareOp operator, byte[] value, Filter filter) {
+ this.operator = operator;
+ this.value = value;
+ this.filter = filter;
+ }
+
+ public CompareOp getOperator() {
+ return this.operator;
+ }
+
+ public byte[] getValue() {
+ return this.value;
+ }
+
+ public Filter getFilter() {
+ return this.filter;
+ }
+ }
+}
\ No newline at end of file
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterNode.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterNode.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/FilterNode.java (working copy)
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.index.Column;
+import org.apache.hadoop.hbase.index.IndexSpecification;
+import org.apache.hadoop.hbase.util.Pair;
+
+public interface FilterNode {
+ Map, IndexSpecification> getIndexToUse();
+
+ Map>> getPossibleUseIndices();
+
+ Map>> getPossibleFutureUseIndices();
+}
\ No newline at end of file
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexFilterNode.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexFilterNode.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexFilterNode.java (working copy)
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.index.Column;
+import org.apache.hadoop.hbase.index.IndexSpecification;
+import org.apache.hadoop.hbase.util.Pair;
+
+public class IndexFilterNode implements LeafFilterNode {
+
+ private IndexSpecification indexToUse;
+ // all possible indices which can be used. This includes the selected indexToUse also.
+ // This contains the an integer as the second item in the Pair. This is the relative overhead
+ // in scanning the index region. The lesser the value the lesser the overhead in scanning the
+ // index region. This will be set with the number of columns in the index specification.
+ private List> possibleUseIndices;
+
+ private List> possibleFutureUseIndices;
+
+ private FilterColumnValueDetail filterColumnValueDetail;
+
+ @Override
+ public Map>> getPossibleFutureUseIndices() {
+ // TODO avoid create of Map instance all the time...
+ Map>> reply =
+ new HashMap>>();
+ reply.put(filterColumnValueDetail.getColumn(), possibleFutureUseIndices);
+ return reply;
+ }
+
+ public IndexFilterNode(IndexSpecification indexToUse,
+ List> possibleUseIndices,
+ List> possibleFutureUseIndices,
+ FilterColumnValueDetail filterColumnValueDetail) {
+ this.indexToUse = indexToUse;
+ this.possibleUseIndices = possibleUseIndices;
+ this.possibleFutureUseIndices = possibleFutureUseIndices;
+ this.filterColumnValueDetail = filterColumnValueDetail;
+ }
+
+ /**
+ * all possible indices which can be used. This includes the selected indexToUse also. This
+ * contains the an integer as the second item in the Pair. This is the relative overhead in
+ * scanning the index region. The lesser the value the lesser the overhead in scanning the index
+ * region. This will be set with the number of columns in the index specification.
+ * @return all possible indices can be used.
+ */
+ @Override
+ public Map>> getPossibleUseIndices() {
+ // TODO avoid create of Map instance all the time...
+ Map>> reply =
+ new HashMap>>();
+ reply.put(filterColumnValueDetail.getColumn(), possibleUseIndices);
+ return reply;
+ }
+
+ @Override
+ public Map, IndexSpecification> getIndexToUse() {
+ // TODO avoid create of Map instance all the time...
+ Map, IndexSpecification> reply =
+ new HashMap, IndexSpecification>();
+ List key = new ArrayList(1);
+ key.add(filterColumnValueDetail);
+ reply.put(key, indexToUse);
+ return reply;
+ }
+
+ @Override
+ public IndexSpecification getBestIndex() {
+ return this.indexToUse;
+ }
+
+ @Override
+ public FilterColumnValueDetail getFilterColumnValueDetail() {
+ return this.filterColumnValueDetail;
+ }
+
+ public void setFilterColumnValueDetail(FilterColumnValueDetail filterColumnValueDetail) {
+ this.filterColumnValueDetail = filterColumnValueDetail;
+ }
+}
\ No newline at end of file
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionObserver.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionObserver.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionObserver.java (working copy)
@@ -0,0 +1,953 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.index.ColumnQualifier;
+import org.apache.hadoop.hbase.index.Constants;
+import org.apache.hadoop.hbase.index.IndexSpecification;
+import org.apache.hadoop.hbase.index.TableIndices;
+import org.apache.hadoop.hbase.index.io.IndexHalfStoreFileReader;
+import org.apache.hadoop.hbase.index.manager.IndexManager;
+import org.apache.hadoop.hbase.index.util.ByteArrayBuilder;
+import org.apache.hadoop.hbase.index.util.IndexUtils;
+import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
+import org.apache.hadoop.hbase.io.Reference;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegion.Operation;
+import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
+import org.apache.hadoop.hbase.regionserver.StoreFile.Reader;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
+import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
+import org.apache.hadoop.hbase.regionserver.OperationStatus;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.SplitTransaction;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.PairOfSameType;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+public class IndexRegionObserver extends BaseRegionObserver {
+
+ private static final Log LOG = LogFactory.getLog(IndexRegionObserver.class);
+
+ // variable will be set to true in test case for testing
+ // All below public static fields are used for testing.
+ static boolean isTestingEnabled = false;
+
+ public static boolean isSeekpointAddded = false;
+
+ public static boolean isIndexedFlowUsed = false;
+
+ public static List seekPoints = null;
+
+ public static List seekPointsForMultipleIndices = null;
+
+ private Map scannerMap =
+ new ConcurrentHashMap();
+
+ private IndexManager indexManager = IndexManager.getInstance();
+
+ public static final ThreadLocal threadLocal = new ThreadLocal() {
+ @Override
+ protected IndexEdits initialValue() {
+ return new IndexEdits();
+ }
+ };
+
+ @Override
+ public void postOpen(ObserverContext contx) {
+ HTableDescriptor tableDesc = contx.getEnvironment().getRegion().getTableDesc();
+ RegionServerServices rss = contx.getEnvironment().getRegionServerServices();
+ TableName tableName = tableDesc.getTableName();
+ if (isNotIndexedTableDescriptor(tableDesc)) {
+ return;
+ }
+ LOG.trace("Entering postOpen for the table " + tableName);
+ this.indexManager.incrementRegionCount(tableName.getNameAsString());
+ List list = indexManager.getIndicesForTable(tableName.getNameAsString());
+ if (null != list) {
+ LOG.trace("Index Manager already contains an entry for the table "
+ + ". Hence returning from postOpen");
+ return;
+ }
+ byte[] indexBytes = tableDesc.getValue(Constants.INDEX_SPEC_KEY);
+ TableIndices tableIndices = new TableIndices();
+ try {
+ tableIndices.readFields(indexBytes);
+ } catch (IOException e) {
+ rss.abort("Some unidentified scenario while reading from the "
+ + "table descriptor . Aborting RegionServer", e);
+ }
+ list = tableIndices.getIndices();
+ if (list != null && list.size() > 0) {
+ indexManager.addIndexForTable(tableName.getNameAsString(), list);
+ LOG.trace("Added index Specification in the Manager for the " + tableName);
+ }
+ LOG.trace("Exiting postOpen for the table " + tableName);
+ }
+
+ @Override
+ public void preBatchMutate(final ObserverContext ctx,
+ final MiniBatchOperationInProgress miniBatchOp) throws IOException {
+ HRegionServer rs = (HRegionServer) ctx.getEnvironment().getRegionServerServices();
+ HRegion userRegion = ctx.getEnvironment().getRegion();
+ HTableDescriptor userTableDesc = userRegion.getTableDesc();
+ String tableName = userTableDesc.getNameAsString();
+ if (isNotIndexedTableDescriptor(userTableDesc)) {
+ return;
+ }
+ List indices = indexManager.getIndicesForTable(tableName);
+ if (indices == null || indices.isEmpty()) {
+ LOG.trace("skipping preBatchMutate for the table " + tableName + " as there are no indices");
+ return;
+ }
+ LOG.trace("Entering preBatchMutate for the table " + tableName);
+ LOG.trace("Indices for the table " + tableName + " are: " + indices);
+ HRegion indexRegion = getIndexTableRegion(tableName, userRegion, rs);
+ // Storing this found HRegion in the index table within the thread locale.
+ IndexEdits indexEdits = threadLocal.get();
+ indexEdits.indexRegion = indexRegion;
+ for (int i = 0; i < miniBatchOp.size(); i++) {
+ Mutation m = miniBatchOp.getOperation(i);
+ if (m instanceof Put) {
+ try {
+ prepareIndexMutations(indices, userRegion, m, tableName, indexRegion);
+ } catch (IOException e) {
+ miniBatchOp.setOperationStatus(i, new OperationStatus(
+ OperationStatusCode.SANITY_CHECK_FAILURE, e.getMessage()));
+ }
+ } else if (m instanceof Delete) {
+ prepareIndexMutations(indices, userRegion, m, tableName, indexRegion);
+ }
+ }
+ indexEdits.setUpdateLocked();
+ indexRegion.updatesLock();
+ LOG.trace("Exiting preBatchMutate for the table " + tableName);
+ }
+
+ // TODO make this api generic and use in scanner open hook to find the index region.
+ private HRegion getIndexTableRegion(String tableName, HRegion userRegion, HRegionServer rs)
+ throws IOException {
+ TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName));
+ Collection idxTabRegions = rs.getOnlineRegions(indexTableName);
+ for (HRegion idxTabRegion : idxTabRegions) {
+ // TODO start key check is enough? May be we can check for the
+ // possibility for N-1 Mapping?
+ if (Bytes.equals(idxTabRegion.getStartKey(), userRegion.getStartKey())) {
+ return idxTabRegion;
+ }
+ }
+ // No corresponding index region found in the RS online regions list!
+ String message =
+ "Index Region not found on the region server . " + "So skipping the put. Need Balancing";
+ LOG.warn(message);
+ // TODO give a proper Exception msg
+ throw new DoNotRetryIOException(message);
+ }
+
+ private void prepareIndexMutations(List indices, HRegion userRegion,
+ Mutation mutation, String tableName, HRegion indexRegion) throws IOException {
+ IndexEdits indexEdits = threadLocal.get();
+ if (mutation instanceof Put) {
+ for (IndexSpecification index : indices) {
+ // Handle each of the index
+ Mutation indexPut = IndexUtils.prepareIndexPut((Put) mutation, index, indexRegion);
+ if (null != indexPut) {
+ // This mutation can be null when the user table mutation is not
+ // containing all of the indexed col value.
+ indexEdits.add(indexPut);
+ }
+ }
+ } else if (mutation instanceof Delete) {
+ Collection extends Mutation> indexDeletes =
+ prepareIndexDeletes((Delete) mutation, userRegion, indices, indexRegion);
+ indexEdits.addAll(indexDeletes);
+ } else {
+ // TODO : Log or throw exception
+ }
+ }
+
+ Collection extends Mutation> prepareIndexDeletes(Delete delete, HRegion userRegion,
+ List indexSpecs, HRegion indexRegion) throws IOException {
+ Collection indexDeletes = new LinkedHashSet();
+ for (Entry> entry : delete.getFamilyCellMap().entrySet()) {
+ for (Cell cell : entry.getValue()) {
+ indexDeletes.addAll(getIndexDeletes(indexSpecs, userRegion, indexRegion,
+ KeyValueUtil.ensureKeyValue(cell)));
+ }
+ }
+ return indexDeletes;
+ }
+
+ private static Collection getIndexDeletes(List indexSpecs,
+ HRegion userRegion, HRegion indexRegion, Cell deleteKV) throws IOException {
+ Collection indexDeletes = new LinkedHashSet();
+ List indicesToUpdate = new LinkedList();
+ Multimap groupedKV =
+ doGetAndGroupByTS(indexSpecs, userRegion, deleteKV, indicesToUpdate);
+
+ // There can be multiple index kvs for each user kv
+ // So, prepare all resultant index delete kvs for this user delete kv
+ for (Entry> entry : groupedKV.asMap().entrySet()) {
+ for (IndexSpecification index : indicesToUpdate) {
+ ByteArrayBuilder indexRow =
+ IndexUtils.getIndexRowKeyHeader(index, indexRegion.getStartKey(), deleteKV.getRow());
+ boolean update = false;
+ for (ColumnQualifier cq : index.getIndexColumns()) {
+ Cell kvFound = null;
+ for (Cell kv : entry.getValue()) {
+ if (Bytes.equals(cq.getColumnFamily(), kv.getFamily())
+ && Bytes.equals(cq.getQualifier(), kv.getQualifier())) {
+ kvFound = kv;
+ update = true;
+ break;
+ }
+ }
+ if (kvFound == null) {
+ indexRow.position(indexRow.position() + cq.getMaxValueLength());
+ } else {
+ IndexUtils.updateRowKeyForKV(cq, kvFound, indexRow);
+ }
+ }
+ if (update) {
+ // Append the actual row key at the end of the index row key.
+ indexRow.put(deleteKV.getRow());
+ Delete idxDelete = new Delete(indexRow.array());
+ if (((KeyValue) deleteKV).isDeleteType()) {
+ idxDelete
+ .deleteColumn(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, entry.getKey());
+ } else {
+ idxDelete.deleteFamily(Constants.IDX_COL_FAMILY, entry.getKey());
+ }
+ indexDeletes.add(idxDelete);
+ }
+ }
+ }
+ return indexDeletes;
+ }
+
+ private static Multimap doGetAndGroupByTS(List indexSpecs,
+ HRegion userRegion, Cell deleteKV, List indicesToConsider)
+ throws IOException {
+
+ Get get = new Get(deleteKV.getRow());
+ long maxTS = HConstants.LATEST_TIMESTAMP;
+
+ if (deleteKV.getTimestamp() < maxTS) {
+ // Add +1 to make the current get includes the timestamp
+ maxTS = deleteKV.getTimestamp() + 1;
+ }
+ get.setTimeRange(0L, maxTS);
+
+ for (IndexSpecification index : indexSpecs) {
+ // Get all indices involves this family/qualifier
+ if (index.contains(deleteKV.getFamily(), deleteKV.getQualifier())) {
+ indicesToConsider.add(index);
+ for (ColumnQualifier cq : index.getIndexColumns()) {
+ get.addColumn(cq.getColumnFamily(), cq.getQualifier());
+ }
+ }
+ }
+ if (((KeyValue) deleteKV).isDeleteType()) {
+ get.setMaxVersions(1);
+ } else if (((KeyValue) deleteKV).isDeleteColumnOrFamily()) {
+ get.setMaxVersions();
+ }
+ List userKVs = userRegion.get(get).list();
+
+ // Group KV based on timestamp
+ Multimap groupedKV = HashMultimap.create();
+
+ if (userKVs != null) {
+ for (Cell userKV : userKVs) {
+ groupedKV.put(userKV.getTimestamp(), userKV);
+ }
+ }
+ return groupedKV;
+ }
+
+ // collection of edits for index table's memstore and WAL
+ public static class IndexEdits {
+ private WALEdit walEdit = new WALEdit();
+ private HRegion indexRegion;
+ private boolean updatesLocked = false;
+
+ /**
+ * Collection of mutations with locks. Locks will be null always as they not yet acquired for
+ * index table.
+ * @see HRegion#batchMutate(Pair[])
+ */
+ private List mutations = new ArrayList();
+
+ public WALEdit getWALEdit() {
+ return this.walEdit;
+ }
+
+ public boolean isUpdatesLocked() {
+ return this.updatesLocked;
+ }
+
+ public void setUpdateLocked() {
+ updatesLocked = true;
+ }
+
+ public void add(Mutation mutation) {
+ // Check if WAL is disabled
+ for (List extends Cell> kvs : mutation.getFamilyCellMap().values()) {
+ for (Cell cell : kvs) {
+ this.walEdit.add(KeyValueUtil.ensureKeyValue(cell));
+ }
+ }
+ // There is no lock acquired for index table. So, set it to null
+ this.mutations.add(mutation);
+ }
+
+ public void addAll(Collection extends Mutation> mutations) {
+ for (Mutation mutation : mutations) {
+ add(mutation);
+ }
+ }
+
+ public List getIndexMutations() {
+ return this.mutations;
+ }
+
+ public HRegion getRegion() {
+ return this.indexRegion;
+ }
+ }
+
+ @Override
+ public void postBatchMutate(final ObserverContext ctx,
+ final MiniBatchOperationInProgress miniBatchOp) {
+ HTableDescriptor userTableDesc = ctx.getEnvironment().getRegion().getTableDesc();
+ String tableName = userTableDesc.getNameAsString();
+ if (isNotIndexedTableDescriptor(userTableDesc)) {
+ return;
+ }
+ List indices = indexManager.getIndicesForTable(tableName);
+ if (indices == null || indices.isEmpty()) {
+ LOG.trace("skipping postBatchMutate for the table " + tableName + " as there are no indices");
+ return;
+ }
+ LOG.trace("Entering postBatchMutate for the table " + tableName);
+ IndexEdits indexEdits = threadLocal.get();
+ List indexMutations = indexEdits.getIndexMutations();
+
+ if (indexMutations.size() == 0) {
+ return;
+ }
+ HRegion hr = indexEdits.getRegion();
+ try {
+ hr.batchMutateForIndex(indexMutations.toArray(new Mutation[indexMutations.size()]));
+ } catch (IOException e) {
+ // TODO This can come? If so we need to revert the actual put
+ // and make the op failed.
+ LOG.error("Error putting data into the index region", e);
+ }
+ LOG.trace("Exiting postBatchMutate for the table " + tableName);
+ }
+
+ private boolean isNotIndexedTableDescriptor(HTableDescriptor userTableDesc) {
+ TableName tableName = userTableDesc.getTableName();
+ return IndexUtils.isCatalogOrSystemTable(tableName)
+ || IndexUtils.isIndexTable(tableName)
+ || (!IndexUtils.isIndexTable(tableName) && userTableDesc.getValue(Constants.INDEX_SPEC_KEY) == null);
+ }
+
+ @Override
+ public void postBatchMutateIndispensably(ObserverContext ctx,
+ MiniBatchOperationInProgress miniBatchOp, boolean success) throws IOException {
+ IndexEdits indexEdits = threadLocal.get();
+ if (indexEdits != null) {
+ if (indexEdits.isUpdatesLocked()) {
+ indexEdits.getRegion().updatesUnlock();
+ }
+ }
+ threadLocal.remove();
+ }
+
+ @Override
+ public boolean postScannerFilterRow(ObserverContext e,
+ InternalScanner s, byte[] currentRow, final int offset, final short length, boolean hasMore)
+ throws IOException {
+ HTableDescriptor tableDesc = e.getEnvironment().getRegion().getTableDesc();
+ if (isNotIndexedTableDescriptor(tableDesc)) {
+ return hasMore;
+ }
+ SeekAndReadRegionScanner bsrs = SeekAndReadRegionScannerHolder.getRegionScanner();
+ if (bsrs != null) {
+ while (false == bsrs.seekToNextPoint()) {
+ SeekPointFetcher seekPointFetcher = scannerMap.get(bsrs);
+ if (null != seekPointFetcher) {
+ List seekPoints = new ArrayList(1);
+ seekPointFetcher.nextSeekPoints(seekPoints, 1);
+ // TODO use return boolean?
+ if (seekPoints.isEmpty()) {
+ LOG.trace("No seekpoints are remaining hence returning.. ");
+ return false;
+ }
+ bsrs.addSeekPoints(seekPoints);
+ if (isTestingEnabled) {
+ setSeekPoints(seekPoints);
+ setSeekpointAdded(true);
+ addSeekPoints(seekPoints);
+ }
+ } else {
+ // This will happen for a region with no index
+ break;
+ }
+ }
+ }
+ return true;
+ }
+
+ public RegionScanner postScannerOpen(ObserverContext e, Scan scan,
+ RegionScanner s) {
+ HRegion region = e.getEnvironment().getRegion();
+ HTableDescriptor tableDesc = region.getTableDesc();
+ String tableName = tableDesc.getNameAsString();
+ HRegionServer rs = (HRegionServer) e.getEnvironment().getRegionServerServices();
+ if (isNotIndexedTableDescriptor(tableDesc)) {
+ return s;
+ }
+ // If the passed region is a region from an indexed table
+ SeekAndReadRegionScanner bsrs = null;
+
+ try {
+ List indexlist = IndexManager.getInstance().getIndicesForTable(tableName);
+ if (indexlist == null || indexlist.isEmpty()) {
+ // Not an indexed table. Just return.
+ return s;
+ }
+ if (indexlist != null) {
+ LOG.info("Entering postScannerOpen for the table " + tableName);
+ Collection onlineRegions = ((HRegionServer) rs).getOnlineRegionsLocalContext();
+ for (HRegion onlineIdxRegion : onlineRegions) {
+ if (IndexUtils.isCatalogOrSystemTable(onlineIdxRegion.getTableDesc().getTableName())) {
+ continue;
+ }
+ if (onlineIdxRegion.equals(region)) {
+ continue;
+ }
+ if (Bytes.equals(onlineIdxRegion.getStartKey(), region.getStartKey())
+ && Bytes.equals(Bytes.toBytes(IndexUtils.getIndexTableName(region.getTableDesc()
+ .getNameAsString())), onlineIdxRegion.getTableDesc().getName())) {
+ ScanFilterEvaluator mapper = new ScanFilterEvaluator();
+ IndexRegionScanner indexScanner =
+ mapper.evaluate(scan, indexlist, onlineIdxRegion.getStartKey(), onlineIdxRegion,
+ tableName);
+ if (indexScanner == null) return s;
+ SeekPointFetcher spf = new SeekPointFetcher(indexScanner);
+ ReInitializableRegionScanner reinitializeScanner =
+ new ReInitializableRegionScannerImpl(s, scan, spf);
+ bsrs = new BackwardSeekableRegionScanner(reinitializeScanner, scan, region, null);
+ scannerMap.put(bsrs, spf);
+ LOG.trace("Scanner Map has " + scannerMap);
+ break;
+ }
+ }
+ LOG.trace("Exiting postScannerOpen for the table " + tableName);
+ }
+ } catch (Exception ex) {
+ LOG.error("Exception occured in postScannerOpen for the table " + tableName, ex);
+ }
+ if (bsrs != null) {
+ return bsrs;
+ } else {
+ return s;
+ }
+ }
+
+ public boolean preScannerNext(ObserverContext e, InternalScanner s,
+ List results, int nbRows, boolean hasMore) throws IOException {
+ HRegion region = e.getEnvironment().getRegion();
+ String tableName = region.getTableDesc().getNameAsString();
+ try {
+ if (s instanceof SeekAndReadRegionScanner) {
+ LOG.trace("Entering preScannerNext for the table " + tableName);
+ BackwardSeekableRegionScanner bsrs = (BackwardSeekableRegionScanner) s;
+ SeekAndReadRegionScannerHolder.setRegionScanner(bsrs);
+ SeekPointFetcher spf = scannerMap.get(bsrs);
+ List seekPoints = null;
+ if (spf != null) {
+ if (isTestingEnabled) {
+ setIndexedFlowUsed(true);
+ }
+ seekPoints = new ArrayList();
+ spf.nextSeekPoints(seekPoints, nbRows);
+ }
+ if (seekPoints == null || seekPoints.isEmpty()) {
+ LOG.trace("No seekpoints are remaining hence returning.. ");
+ SeekAndReadRegionScannerHolder.removeRegionScanner();
+ e.bypass();
+ return false;
+ }
+ bsrs.addSeekPoints(seekPoints);
+ // This setting is just for testing purpose
+ if (isTestingEnabled) {
+ setSeekPoints(seekPoints);
+ setSeekpointAdded(true);
+ addSeekPoints(seekPoints);
+ }
+ LOG.trace("Exiting preScannerNext for the table " + tableName);
+ }
+ } catch (Exception ex) {
+ LOG.error("Exception occured in preScannerNext for the table " + tableName + ex);
+ }
+ return true;
+ }
+
+ @Override
+ public boolean postScannerNext(ObserverContext e,
+ InternalScanner s, List results, int limit, boolean hasMore) throws IOException {
+ if (s instanceof SeekAndReadRegionScanner) {
+ SeekAndReadRegionScannerHolder.removeRegionScanner();
+ }
+ return true;
+ }
+
+ @Override
+ public void preScannerClose(ObserverContext e, InternalScanner s)
+ throws IOException {
+ if (s instanceof BackwardSeekableRegionScanner) {
+ scannerMap.remove((RegionScanner) s);
+ }
+ }
+
+ @Override
+ public void preSplitBeforePONR(ObserverContext e, byte[] splitKey,
+ List metaEntries) throws IOException {
+ RegionCoprocessorEnvironment environment = e.getEnvironment();
+ HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
+ HRegion region = environment.getRegion();
+ String userTableName = region.getTableDesc().getNameAsString();
+ LOG.trace("Entering preSplitBeforePONR for the table " + userTableName + " for the region "
+ + region.getRegionInfo());
+ String indexTableName = IndexUtils.getIndexTableName(userTableName);
+ TableIndices tableIndices = new TableIndices();
+ if (indexManager.getIndicesForTable(userTableName) == null) {
+ byte[] indexBytes = region.getTableDesc().getValue(Constants.INDEX_SPEC_KEY);
+ if (indexBytes == null) {
+ return;
+ }
+ tableIndices.readFields(indexBytes);
+ indexManager.addIndexForTable(userTableName, tableIndices.getIndices());
+ }
+ if (indexManager.getIndicesForTable(userTableName) != null) {
+ HRegion indexRegion = null;
+ SplitTransaction st = null;
+ try {
+ indexRegion = getIndexRegion(rs, region.getStartKey(), indexTableName);
+ if (null != indexRegion) {
+ LOG.info("Flushing the cache for the index table " + indexTableName + " for the region "
+ + indexRegion.getRegionInfo());
+ indexRegion.flushcache();
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Forcing split for the index table " + indexTableName + " with split key "
+ + Bytes.toString(splitKey));
+ }
+ st = new SplitTransaction(indexRegion, splitKey);
+ if (!st.prepare()) {
+ LOG.error("Prepare for the index table " + indexTableName
+ + " failed. So returning null. ");
+ e.bypass();
+ return;
+ }
+ indexRegion.forceSplit(splitKey);
+ PairOfSameType daughterRegions = st.stepsBeforePONR(rs, rs, false);
+ splitThreadLocal.set(new SplitInfo(indexRegion, daughterRegions, st));
+ HRegionInfo copyOfParent = new HRegionInfo(region.getRegionInfo());
+ copyOfParent.setOffline(true);
+ copyOfParent.setSplit(true);
+ // Put for parent
+ Put putParent = MetaEditor.makePutFromRegionInfo(copyOfParent);
+ MetaEditor.addDaughtersToPut(putParent, daughterRegions.getFirst().getRegionInfo(),
+ daughterRegions.getSecond().getRegionInfo());
+ metaEntries.add(putParent);
+ // Puts for daughters
+ Put putA = MetaEditor.makePutFromRegionInfo(daughterRegions.getFirst().getRegionInfo());
+ Put putB = MetaEditor.makePutFromRegionInfo(daughterRegions.getSecond().getRegionInfo());
+ st.addLocation(putA, rs.getServerName(), 1);
+ st.addLocation(putB, rs.getServerName(), 1);
+ metaEntries.add(putA);
+ metaEntries.add(putB);
+ LOG.info("Daughter regions created for the index table " + indexTableName
+ + " for the region " + indexRegion.getRegionInfo());
+ return;
+ } else {
+ LOG.error("IndexRegion for the table " + indexTableName + " is null. So returning null. ");
+ e.bypass();
+ return;
+ }
+ } catch (Exception ex) {
+ LOG.error("Error while spliting the indexTabRegion or not able to get the indexTabRegion:"
+ + indexRegion != null ? indexRegion.getRegionName() : "", ex);
+ if (st != null) st.rollback(rs, rs);
+ e.bypass();
+ return;
+ }
+ }
+ }
+
+ @Override
+ public void preSplit(ObserverContext e) throws IOException {
+ if (splitThreadLocal != null) {
+ splitThreadLocal.remove();
+ }
+ }
+
+ private HRegion getIndexRegion(HRegionServer rs, byte[] startKey, String indexTableName)
+ throws IOException {
+ List indexTabRegions = rs.getOnlineRegions(TableName.valueOf(indexTableName));
+ for (HRegion indexRegion : indexTabRegions) {
+ if (Bytes.equals(startKey, indexRegion.getStartKey())) {
+ return indexRegion;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public void preSplitAfterPONR(ObserverContext e) throws IOException {
+ RegionCoprocessorEnvironment environment = e.getEnvironment();
+ HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
+ HRegion region = environment.getRegion();
+ String userTableName = region.getTableDesc().getNameAsString();
+ String indexTableName = IndexUtils.getIndexTableName(userTableName);
+ if (IndexUtils.isIndexTable(userTableName)) {
+ return;
+ }
+ LOG.trace("Entering postSplit for the table " + userTableName + " for the region "
+ + region.getRegionInfo());
+ SplitTransaction splitTransaction = null;
+ if (region.getTableDesc().getValue(Constants.INDEX_SPEC_KEY) != null) {
+ try {
+ SplitInfo splitInfo = splitThreadLocal.get();
+ if (splitInfo == null) return;
+ splitTransaction = splitInfo.getSplitTransaction();
+ PairOfSameType daughters = splitInfo.getDaughters();
+ if (splitTransaction != null && daughters != null) {
+ splitTransaction.stepsAfterPONR(rs, rs, daughters);
+ LOG.info("Daughter regions are opened and split transaction finished for zknodes for index table "
+ + indexTableName + " for the region " + region.getRegionInfo());
+ }
+ } catch (Exception ex) {
+ String msg =
+ "Splitting of index region has failed in stepsAfterPONR stage so aborting the server";
+ LOG.error(msg, ex);
+ rs.abort(msg);
+ }
+ }
+ }
+
+ // A thread local variable used to get the splitted region information of the index region.
+ // This is needed becuase in order to do the PONR entry we need the info of the index
+ // region's daughter entries.
+ public static final ThreadLocal splitThreadLocal = new ThreadLocal() {
+ protected SplitInfo initialValue() {
+ return null;
+ };
+ };
+
+ @Override
+ public void preRollBackSplit(ObserverContext ctx)
+ throws IOException {
+ RegionCoprocessorEnvironment environment = ctx.getEnvironment();
+ HRegionServer rs = (HRegionServer) environment.getRegionServerServices();
+ HRegion region = environment.getRegion();
+ HTableDescriptor tableDesc = region.getTableDesc();
+ String userTableName = tableDesc.getNameAsString();
+ if (isNotIndexedTableDescriptor(tableDesc)) {
+ return;
+ }
+ LOG.trace("Entering preRollBack for the table " + userTableName + " for the region "
+ + region.getRegionInfo());
+ SplitInfo splitInfo = splitThreadLocal.get();
+ if (splitInfo == null) return;
+ SplitTransaction splitTransaction = splitInfo.getSplitTransaction();
+ try {
+ if (splitTransaction != null) {
+ splitTransaction.rollback(rs, rs);
+ LOG.info("preRollBack successfully done for the table " + userTableName
+ + " for the region " + region.getRegionInfo());
+ }
+ } catch (Exception e) {
+ LOG.error(
+ "Error while rolling back the split failure for index region " + splitInfo.getParent(), e);
+ rs.abort("Abort; we got an error during rollback of index");
+ }
+ }
+
+ // For testing to check whether final step of seek point is added
+ public static void setSeekpointAdded(boolean isSeekpointAddded) {
+ IndexRegionObserver.isSeekpointAddded = isSeekpointAddded;
+ }
+
+ // For testing
+ public static boolean getSeekpointAdded() {
+ return isSeekpointAddded;
+ }
+
+ // For testing to ensure indexed flow is used or not
+ public static void setIndexedFlowUsed(boolean isIndexedFlowUsed) {
+ IndexRegionObserver.isIndexedFlowUsed = isIndexedFlowUsed;
+ }
+
+ // For testing
+ public static boolean getIndexedFlowUsed() {
+ return isIndexedFlowUsed;
+ }
+
+ // For testing
+ static List getSeekpoints() {
+ return seekPoints;
+ }
+
+ // For testing to ensure cache size is returned correctly
+ public static void setSeekPoints(List seekPoints) {
+ IndexRegionObserver.seekPoints = seekPoints;
+ }
+
+ public static void setIsTestingEnabled(boolean isTestingEnabled) {
+ IndexRegionObserver.isTestingEnabled = isTestingEnabled;
+ }
+
+ public synchronized static void addSeekPoints(List seekPoints) {
+ if (seekPoints == null) {
+ IndexRegionObserver.seekPointsForMultipleIndices = null;
+ return;
+ }
+ if (IndexRegionObserver.seekPointsForMultipleIndices == null) {
+ IndexRegionObserver.seekPointsForMultipleIndices = new ArrayList();
+ }
+ IndexRegionObserver.seekPointsForMultipleIndices.addAll(seekPoints);
+ }
+
+ public static List getMultipleSeekPoints() {
+ return IndexRegionObserver.seekPointsForMultipleIndices;
+ }
+
+ private static class SeekAndReadRegionScannerHolder {
+ private static ThreadLocal holder =
+ new ThreadLocal();
+
+ public static void setRegionScanner(SeekAndReadRegionScanner scanner) {
+ holder.set(scanner);
+ }
+
+ public static SeekAndReadRegionScanner getRegionScanner() {
+ return holder.get();
+ }
+
+ public static void removeRegionScanner() {
+ holder.remove();
+ }
+ }
+
+ @Override
+ public InternalScanner preCompactScannerOpen(ObserverContext c,
+ Store store, List extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs,
+ InternalScanner s, CompactionRequest request) throws IOException {
+ HRegionServer rs = (HRegionServer) c.getEnvironment().getRegionServerServices();
+ if (!IndexUtils.isIndexTable(store.getTableName())) {
+ // Not an index table
+ return null;
+ }
+ long smallestReadPoint = c.getEnvironment().getRegion().getSmallestReadPoint();
+ String actualTableName = IndexUtils.getActualTableName(store.getTableName().getNameAsString());
+ TTLStoreScanner ttlStoreScanner =
+ new TTLStoreScanner(store, smallestReadPoint, earliestPutTs, scanType, scanners,
+ new TTLExpiryChecker(), actualTableName, rs);
+ return ttlStoreScanner;
+ }
+
+ @Override
+ public void postClose(ObserverContext e, boolean abortRequested) {
+ HRegion region = e.getEnvironment().getRegion();
+ byte[] tableName = region.getRegionInfo().getTable().getName();
+ if (IndexUtils.isCatalogOrSystemTable(region.getTableDesc().getTableName())
+ || IndexUtils.isIndexTable(tableName)) {
+ return;
+ }
+ if (splitThreadLocal.get() == null) {
+ this.indexManager.decrementRegionCount(Bytes.toString(tableName), true);
+ } else {
+ this.indexManager.decrementRegionCount(Bytes.toString(tableName), false);
+ }
+ }
+
+ private boolean isValidIndexMutation(HTableDescriptor userTableDesc) {
+ String tableName = userTableDesc.getTableName().getNameAsString();
+ if (IndexUtils.isCatalogOrSystemTable(userTableDesc.getTableName())
+ || IndexUtils.isIndexTable(tableName)) {
+ return false;
+ }
+ List indices = indexManager.getIndicesForTable(tableName);
+ if (indices == null || indices.isEmpty()) {
+ LOG.trace("skipping preBatchMutate for the table " + tableName + " as there are no indices");
+ return false;
+ }
+ return true;
+ }
+
+ private void acquireLockOnIndexRegion(String tableName, HRegion userRegion, HRegionServer rs,
+ Operation op) throws IOException {
+ HRegion indexRegion = getIndexTableRegion(tableName, userRegion, rs);
+ indexRegion.checkResources();
+ indexRegion.startRegionOperation(op);
+ }
+
+ @Override
+ public void postCloseRegionOperation(ObserverContext ctx,
+ Operation op) throws IOException {
+ if (op.equals(Operation.BATCH_MUTATE)) {
+ HRegionServer rs = (HRegionServer) ctx.getEnvironment().getRegionServerServices();
+ HRegion userRegion = ctx.getEnvironment().getRegion();
+ HTableDescriptor userTableDesc = userRegion.getTableDesc();
+ String tableName = userTableDesc.getNameAsString();
+ if (isNotIndexedTableDescriptor(userTableDesc)) {
+ return;
+ }
+ if (!isValidIndexMutation(userTableDesc)) {
+ // Ideally need not release any lock because in the preStartRegionOperationHook we would not
+ // have
+ // acquired
+ // any lock on the index region
+ return;
+ }
+ HRegion indexRegion = getIndexTableRegion(tableName, userRegion, rs);
+ // This check for isClosed and isClosing is needed because we should not unlock
+ // when the index region lock would have already been released before throwing NSRE
+
+ // TODO : What is the scenario that i may get an IllegalMonitorStateException
+ if (!indexRegion.isClosed() || !indexRegion.isClosing()) {
+ indexRegion.closeRegionOperation();
+ }
+ }
+ }
+
+ @Override
+ public void
+ postStartRegionOperation(ObserverContext e, Operation op)
+ throws IOException {
+ if (op.equals(Operation.BATCH_MUTATE)) {
+ HRegionServer rs = (HRegionServer) e.getEnvironment().getRegionServerServices();
+ HRegion userRegion = e.getEnvironment().getRegion();
+ HTableDescriptor userTableDesc = userRegion.getTableDesc();
+ String tableName = userTableDesc.getNameAsString();
+ if (isNotIndexedTableDescriptor(userTableDesc)) {
+ return;
+ }
+ if (!isValidIndexMutation(userTableDesc)) {
+ return;
+ }
+ acquireLockOnIndexRegion(tableName, userRegion, rs, op);
+ }
+ }
+
+ @Override
+ public Reader preStoreFileReaderOpen(ObserverContext ctx,
+ FileSystem fs, Path path, FSDataInputStreamWrapper in, long size, CacheConfig cacheConf,
+ Reference r, Reader reader) throws IOException {
+ Configuration conf = ctx.getEnvironment().getConfiguration();
+ if (reader == null && r != null && isIndexRegionReference(path)) {
+ return new IndexHalfStoreFileReader(fs, path, cacheConf, in, size, r, conf);
+ }
+ return reader;
+ }
+
+ private boolean isIndexRegionReference(Path path) {
+ String tablePath = path.getParent().getParent().getParent().getName();
+ return tablePath.endsWith(Constants.INDEX_TABLE_SUFFIX);
+ }
+
+ /**
+ * Information of dependent region split processed in coprocessor hook
+ * {@link RegionCoprocessorHost#preSplitBeforePONR(byte[], List)}
+ */
+ public static class SplitInfo {
+ private PairOfSameType daughterRegions;
+ private SplitTransaction st;
+ private HRegion parent;
+
+ public SplitInfo(final HRegion parent, final PairOfSameType pairOfSameType,
+ final SplitTransaction st) {
+ this.parent = parent;
+ this.daughterRegions = pairOfSameType;
+ this.st = st;
+ }
+
+ public PairOfSameType getDaughters() {
+ return this.daughterRegions;
+ }
+
+ public SplitTransaction getSplitTransaction() {
+ return this.st;
+ }
+
+ public HRegion getParent() {
+ return this.parent;
+ }
+ }
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScanner.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScanner.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScanner.java (working copy)
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+
+public interface IndexRegionScanner extends RegionScanner {
+
+ public void advance();
+
+ public void setRangeFlag(boolean range);
+
+ public boolean isRange();
+
+ public void setScannerIndex(int index);
+
+ public int getScannerIndex();
+
+ public boolean hasChildScanners();
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForAND.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForAND.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForAND.java (working copy)
@@ -0,0 +1,309 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.index.util.IndexUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+
+public class IndexRegionScannerForAND implements IndexRegionScanner {
+
+ private static final Log LOG = LogFactory.getLog(IndexRegionScannerForAND.class);
+
+ private List scanners = null;
+
+ private Map, Integer>> rowCache =
+ new HashMap, Integer>>();
+
+ private int scannersCount = 0;
+
+ volatile boolean hasRangeScanners = false;
+
+ private int scannerIndex = -1;
+
+ public IndexRegionScannerForAND(List scanners) {
+ this.scanners = scanners;
+ scannersCount = scanners.size();
+ }
+
+ @Override
+ public void advance() {
+ for (IndexRegionScanner scn : this.scanners) {
+ scn.advance();
+ }
+ }
+
+ @Override
+ public HRegionInfo getRegionInfo() {
+ return null;
+ }
+
+ @Override
+ public boolean isFilterDone() {
+ return false;
+ }
+
+ @Override
+ public boolean hasChildScanners() {
+ return scannersCount != 0;
+ };
+
+ @Override
+ public void setRangeFlag(boolean range) {
+ hasRangeScanners = range;
+ }
+
+ @Override
+ public boolean isRange() {
+ return this.hasRangeScanners;
+ }
+
+ @Override
+ public void setScannerIndex(int index) {
+ scannerIndex = index;
+ }
+
+ @Override
+ public int getScannerIndex() {
+ return scannerIndex;
+ }
+
+ @Override
+ public synchronized boolean reseek(byte[] row) throws IOException {
+ // ideally reseek on AND may not come as AND cannot be a child of another AND.
+ if (this.scanners.isEmpty()) return false;
+ for (IndexRegionScanner scn : this.scanners) {
+ boolean reseek = scn.reseek(row);
+ if (!reseek) return false;
+ }
+ return true;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ for (IndexRegionScanner scn : this.scanners) {
+ scn.close();
+ }
+ this.scanners.clear();
+ }
+
+ @Override
+ public synchronized boolean next(List results) throws IOException {
+ if (this.scanners != null && !this.scanners.isEmpty()) {
+ List> valueList =
+ new ArrayList>();
+ byte[] maxRowKey = null;
+ while (results.size() < 1) {
+ List intermediateResult = new ArrayList();
+ boolean haveSameRows = true;
+ Cell kv = null;
+ Iterator scnItr = this.scanners.iterator();
+ while (scnItr.hasNext()) {
+ IndexRegionScanner scn = scnItr.next();
+ if (!hasRangeScanners) {
+ if (checkForScanner(valueList, scn.getScannerIndex())) continue;
+ }
+ boolean hasMore = scn.next(intermediateResult);
+ if (!hasRangeScanners) {
+ if (intermediateResult != null && !intermediateResult.isEmpty()) {
+ byte[] rowKey = IndexUtils.getRowKeyFromKV(intermediateResult.get(0));
+ if (maxRowKey == null) {
+ maxRowKey = rowKey;
+ } else {
+ int result = Bytes.compareTo(maxRowKey, rowKey);
+ if (haveSameRows) haveSameRows = (result == 0);
+ maxRowKey = result > 0 ? maxRowKey : rowKey;
+ }
+ if (kv == null) kv = intermediateResult.get(0);
+ intermediateResult.clear();
+ valueList.add(new Pair(rowKey, scn));
+ }
+ } else {
+ if (!intermediateResult.isEmpty()) {
+ boolean matching =
+ checkAndPutMatchingEntry(intermediateResult.get(0), scn.getScannerIndex());
+ if (matching) {
+ results.addAll(intermediateResult);
+ }
+ intermediateResult.clear();
+ }
+ }
+ if (!hasMore) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removing scanner " + scn + " from the list.");
+ }
+ scn.close();
+ scnItr.remove();
+ if (hasRangeScanners) {
+ // TODO: we can remove unnecessary rows(which never become matching entries) from
+ // cache on scanner close.
+ if (this.scanners.isEmpty()) return false;
+ }
+ if (results.size() > 0) {
+ break;
+ }
+ }
+ if (results.size() > 0) {
+ return !this.scanners.isEmpty();
+ }
+ }
+ if (!hasRangeScanners) {
+ if (haveSameRows && valueList.size() == scannersCount) {
+ if (kv != null) results.add(kv);
+ return this.scanners.size() == scannersCount;
+ } else if (haveSameRows && valueList.size() != scannersCount) {
+ close();
+ return false;
+ } else {
+ // In case of AND if the reseek on any one scanner returns false
+ // we can close the entire scanners in the AND subtree
+ if (!reseekTheScanners(valueList, maxRowKey)) {
+ close();
+ return false;
+ }
+ }
+ }
+ }
+ if (hasRangeScanners) {
+ return true;
+ } else {
+ return this.scanners.size() == scannersCount;
+ }
+ }
+ return false;
+ }
+
+ private boolean checkForScanner(List> valueList, int scnIndex) {
+ for (Pair value : valueList) {
+ if (value.getSecond().getScannerIndex() == scnIndex) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean checkAndPutMatchingEntry(Cell cell, int index) {
+ String rowKeyFromKV = Bytes.toString(IndexUtils.getRowKeyFromKV(cell));
+ Pair, Integer> countPair = rowCache.get(rowKeyFromKV);
+ if (countPair == null) {
+ // If present scanners count is not equal to actual scanners count,no need to put row key into
+ // cache because this will never become matching entry.
+ if (this.scanners.size() == scannersCount) {
+ List scannerFlags = new ArrayList(scannerIndex);
+ for (int i = 0; i < scannersCount; i++) {
+ scannerFlags.add(false);
+ }
+ countPair = new Pair, Integer>(scannerFlags, 0);
+ // TODO verify
+ // rowCache.put(rowKeyFromKV, countPair);
+ } else {
+ return false;
+ }
+ }
+ Boolean indexFlag = countPair.getFirst().get(index);
+ Integer countObj = countPair.getSecond();
+ // If count is equal to scanner count before increment means its returned already. skip the
+ // result.
+ if (countObj == scannersCount) {
+ return false;
+ }
+ if (!indexFlag) {
+ countObj++;
+ countPair.getFirst().set(index, true);
+ countPair.setSecond(countObj);
+ rowCache.put(rowKeyFromKV, countPair);
+ }
+ if (countObj == scannersCount) {
+ return true;
+ } else {
+ // If the difference between actual scanner count(scannerCount) and number of scanners have
+ // this row(countObj) is more than present scanners size then remove row from cache because
+ // this never be maching entry.
+ if ((scannersCount - countObj) > this.scanners.size()) {
+ rowCache.remove(rowKeyFromKV);
+ return false;
+ }
+ }
+ return false;
+ }
+
+ private boolean reseekTheScanners(List> valueList,
+ byte[] maxRowKey) throws IOException {
+ Iterator> itr = valueList.iterator();
+ while (itr.hasNext()) {
+ Pair rowVsScanner = itr.next();
+ IndexRegionScanner scn = rowVsScanner.getSecond();
+ // We need to call reseek on OR scanner even the last returned row key is equal or more than
+ // max row key to set reseek flag.
+ if (scn instanceof IndexRegionScannerForOR) {
+ rowVsScanner.getSecond().reseek(maxRowKey);
+ itr.remove();
+ continue;
+ }
+ if (Bytes.compareTo(rowVsScanner.getFirst(), maxRowKey) < 0) {
+ if (!scn.reseek(maxRowKey)) {
+ return false;
+ }
+ itr.remove();
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public boolean next(List result, int limit) throws IOException {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public long getMaxResultSize() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public long getMvccReadPoint() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public boolean nextRaw(List result) throws IOException {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean nextRaw(List result, int limit) throws IOException {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForOR.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForOR.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionScannerForOR.java (working copy)
@@ -0,0 +1,315 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.index.util.IndexUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+
+
+public class IndexRegionScannerForOR implements IndexRegionScanner {
+
+ private static final Log LOG = LogFactory.getLog(IndexRegionScannerForOR.class);
+
+ private List scanners = null;
+
+ // private Pair lastReturnedValue = null;
+
+ private volatile byte[] lastReturnedValue = null;
+
+ // Cache to store the values retrieved by range scans
+ private Map rowCache = new HashMap();
+
+ private Map> valueMap =
+ new TreeMap>(Bytes.BYTES_COMPARATOR);
+
+ private volatile boolean hasRangeScanners = false;
+
+ private volatile boolean isRootScanner = false;
+
+ private int scannerIndex = -1;
+
+ private boolean firstScan = true;
+
+ private boolean reseeked = false;
+
+ public IndexRegionScannerForOR(List scanners) {
+ this.scanners = scanners;
+ }
+
+ @Override
+ public void advance() {
+ if (this.lastReturnedValue != null) {
+ // this.lastReturnedValue.getFirst().advance();
+ }
+ this.lastReturnedValue = null;
+ }
+
+ @Override
+ public HRegionInfo getRegionInfo() {
+ return null;
+ }
+
+ @Override
+ public boolean isFilterDone() {
+ return false;
+ }
+
+ @Override
+ public void setRangeFlag(boolean range) {
+ hasRangeScanners = range;
+ }
+
+ public void setRootFlag(boolean isRootScanner) {
+ this.isRootScanner = isRootScanner;
+ }
+
+ @Override
+ public boolean isRange() {
+ return this.hasRangeScanners;
+ }
+
+ @Override
+ public void setScannerIndex(int index) {
+ scannerIndex = index;
+ }
+
+ @Override
+ public int getScannerIndex() {
+ return scannerIndex;
+ }
+
+ @Override
+ public boolean hasChildScanners() {
+ return !scanners.isEmpty();
+ }
+
+ @Override
+ public synchronized boolean reseek(byte[] row) throws IOException {
+ boolean success = false;
+ if (!valueMap.isEmpty()) {
+ Iterator>> itr = valueMap.entrySet().iterator();
+ while (itr.hasNext()) {
+ Entry> entry = itr.next();
+ IndexRegionScanner scn = entry.getValue().getFirst();
+ if (Bytes.compareTo(entry.getKey(), row) < 0) {
+ // If the reseek does not retrieve any row then it means we have reached the end of the
+ // scan. So this scanner can be safely removed.
+ if (!scn.reseek(row)) {
+ removeScanner(scn.getScannerIndex());
+ }
+ itr.remove();
+ } else {
+ break;
+ }
+ }
+ }
+ reseeked = true;
+ this.lastReturnedValue = null;
+ return success;
+ }
+
+ private void removeScanner(int scnIndex) {
+ Iterator itr = this.scanners.iterator();
+ while (itr.hasNext()) {
+ if (itr.next().getScannerIndex() == scnIndex) {
+ itr.remove();
+ break;
+ }
+ }
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ for (IndexRegionScanner scn : this.scanners) {
+ scn.close();
+ }
+ this.valueMap.clear();
+ this.scanners.clear();
+ this.lastReturnedValue = null;
+ }
+
+ @Override
+ public synchronized boolean next(List results) throws IOException {
+ OUTER: while (results.size() < 1) {
+ List intermediateResult = new ArrayList();
+ if (hasRangeScanners || firstScan || reseeked) {
+ Iterator scnItr = this.scanners.iterator();
+ INNER: while (scnItr.hasNext()) {
+ IndexRegionScanner scn = scnItr.next();
+ if (reseeked) {
+ boolean exists = checkForScanner(scn.getScannerIndex());
+ if (exists) continue;
+ }
+ boolean hasMore = scn.next(intermediateResult);
+ if (!hasRangeScanners) {
+ if (intermediateResult != null && intermediateResult.size() > 0) {
+ byte[] rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0));
+ while (valueMap.containsKey(rowKeyFromKV)) {
+ intermediateResult.clear();
+ hasMore = scn.next(intermediateResult);
+ if (!intermediateResult.isEmpty()) {
+ rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0));
+ } else {
+ break;
+ }
+ }
+ if (!hasMore && intermediateResult.isEmpty()) {
+ // Allow other scanners to scan. Nothing to do.
+ } else {
+ valueMap.put(rowKeyFromKV, new Pair(scn,
+ intermediateResult.get(0)));
+ intermediateResult.clear();
+ }
+ }
+ }
+ if (!hasMore) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removing scanner " + scn + " from the list.");
+ }
+ scn.close();
+ scnItr.remove();
+ }
+ if (hasRangeScanners) {
+ if (!intermediateResult.isEmpty()) {
+ String rowKey = Bytes.toString(IndexUtils.getRowKeyFromKV(intermediateResult.get(0)));
+ if (isRootScanner && !rowCache.containsKey(rowKey)) {
+ rowCache.put(rowKey, false);
+ results.addAll(intermediateResult);
+ return !this.scanners.isEmpty();
+ } else if (isRootScanner) {
+ // dont add to results because already exists and scan for other entry.
+ intermediateResult.clear();
+ continue OUTER;
+ } else {
+ results.addAll(intermediateResult);
+ return !this.scanners.isEmpty();
+ }
+ }
+ }
+ }
+ if (firstScan) firstScan = false;
+ if (reseeked) reseeked = false;
+ } else {
+ // Scan on previous scanner which returned minimum values.
+ Entry> minEntry = null;
+ if (!valueMap.isEmpty()) {
+ minEntry = valueMap.entrySet().iterator().next();
+ IndexRegionScanner scn = minEntry.getValue().getFirst();
+ if (Bytes.compareTo(lastReturnedValue, minEntry.getKey()) == 0) {
+ valueMap.remove(minEntry.getKey());
+ boolean hasMore = scn.next(intermediateResult);
+ byte[] rowKeyFromKV = null;
+ if (!intermediateResult.isEmpty()) {
+ rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0));
+ while (valueMap.containsKey(rowKeyFromKV)) {
+ intermediateResult.clear();
+ if (hasMore) {
+ hasMore = minEntry.getValue().getFirst().next(intermediateResult);
+ }
+ if (intermediateResult.isEmpty()) {
+ rowKeyFromKV = null;
+ scn.close();
+ removeScanner(scn.getScannerIndex());
+ break;
+ }
+ rowKeyFromKV = IndexUtils.getRowKeyFromKV(intermediateResult.get(0));
+ }
+ }
+ if (rowKeyFromKV != null) {
+ valueMap.put(rowKeyFromKV,
+ new Pair(scn, intermediateResult.get(0)));
+ intermediateResult.clear();
+ }
+ }
+ } else {
+ return false;
+ }
+ }
+ if (!valueMap.isEmpty()) {
+ Entry> minEntry =
+ valueMap.entrySet().iterator().next();
+ lastReturnedValue = minEntry.getKey();
+ results.add(minEntry.getValue().getSecond());
+ return true;
+ } else {
+ return false;
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(results.size() + " seek points obtained. Values: "
+ + (!results.isEmpty() ? Bytes.toString(results.get(0).getRow()) : 0));
+ }
+ return !results.isEmpty();
+ }
+
+ private boolean checkForScanner(int scnIndex) {
+ Collection> scanners = valueMap.values();
+ for (Pair scn : scanners) {
+ if (scnIndex == scn.getFirst().getScannerIndex()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean next(List result, int limit) throws IOException {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public long getMaxResultSize() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public long getMvccReadPoint() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public boolean nextRaw(List result) throws IOException {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean nextRaw(List result, int limit) throws IOException {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionSplitPolicy.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionSplitPolicy.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/IndexRegionSplitPolicy.java (working copy)
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
+
+/**
+ * Split policy for index regions to avoid split from external requests.
+ */
+public class IndexRegionSplitPolicy extends RegionSplitPolicy {
+
+ @Override
+ protected boolean shouldSplit() {
+ return false;
+ }
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafFilterNode.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafFilterNode.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafFilterNode.java (working copy)
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import org.apache.hadoop.hbase.index.IndexSpecification;
+
+interface LeafFilterNode extends FilterNode {
+ FilterColumnValueDetail getFilterColumnValueDetail();
+
+ void setFilterColumnValueDetail(FilterColumnValueDetail filterColumnValueDetail);
+
+ IndexSpecification getBestIndex();
+}
\ No newline at end of file
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafIndexRegionScanner.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafIndexRegionScanner.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/LeafIndexRegionScanner.java (working copy)
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.index.IndexSpecification;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+
+
+public class LeafIndexRegionScanner implements IndexRegionScanner {
+
+ private static final Log LOG = LogFactory.getLog(LeafIndexRegionScanner.class);
+
+ private RegionScanner delegator = null;
+
+ private Cell currentKV = null;
+
+ private boolean hadMore = true;
+
+ private final IndexSpecification index;
+
+ private volatile boolean isRangeScanner = false;
+
+ private TTLExpiryChecker ttlExpiryChecker;
+
+ private int scannerIndex = -1;
+
+ public LeafIndexRegionScanner(IndexSpecification index, RegionScanner delegator,
+ TTLExpiryChecker ttlExpiryChecker) {
+ this.delegator = delegator;
+ this.index = index;
+ this.ttlExpiryChecker = ttlExpiryChecker;
+ }
+
+ @Override
+ public synchronized void advance() {
+ this.currentKV = null;
+ }
+
+ @Override
+ public HRegionInfo getRegionInfo() {
+ return this.delegator.getRegionInfo();
+ }
+
+ @Override
+ public boolean isFilterDone() throws IOException {
+ return this.delegator.isFilterDone();
+ }
+
+ @Override
+ public void setRangeFlag(boolean range) {
+ isRangeScanner = range;
+ }
+
+ @Override
+ public boolean isRange() {
+ return this.isRangeScanner;
+ }
+
+ @Override
+ public void setScannerIndex(int index) {
+ scannerIndex = index;
+ }
+
+ @Override
+ public int getScannerIndex() {
+ return scannerIndex;
+ }
+
+ @Override
+ public boolean hasChildScanners() {
+ return false;
+ }
+
+ // TODO the passing row to be the full key in the index table.
+ // The callee need to take care of this creation..
+ @Override
+ public synchronized boolean reseek(byte[] row) throws IOException {
+ if (!hadMore) return false;
+ byte[] targetRowKey = createRowKeyForReseek(row);
+ return this.delegator.reseek(targetRowKey);
+ }
+
+ private byte[] createRowKeyForReseek(byte[] targetRow) {
+ byte[] curRK = this.currentKV.getRow();
+ byte[] curValue = this.currentKV.getValue();
+ short actualTabRKOffset = Bytes.toShort(curValue, 2);
+ byte[] newRowKey = new byte[actualTabRKOffset + targetRow.length];
+ System.arraycopy(curRK, 0, newRowKey, 0, actualTabRKOffset);
+ System.arraycopy(targetRow, 0, newRowKey, actualTabRKOffset, targetRow.length);
+ return newRowKey;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ this.delegator.close();
+ }
+
+ @Override
+ public synchronized boolean next(List results) throws IOException {
+ boolean hasMore = false;
+ do {
+ // this check here will prevent extra next call when in the previous
+ // next last row was fetched and after that an advance was called on this
+ // scanner. So instead of making a next call again we can return from here.
+ if (!this.hadMore) return false;
+ hasMore = this.delegator.next(results);
+ if (results != null && results.size() > 0) {
+ Cell kv = results.get(0);
+ if (this.ttlExpiryChecker.checkIfTTLExpired(this.index.getTTL(), kv.getTimestamp())) {
+ results.clear();
+ LOG.info("The ttl has expired for the kv " + kv);
+ } else {
+ if (!isRangeScanner) {
+ // This is need to reseek in case of EQUAL scanners.
+ this.currentKV = kv;
+ break;
+ }
+ }
+ }
+ } while (results != null && results.size() < 1 && hasMore);
+ this.hadMore = hasMore;
+ return hasMore;
+ }
+
+ @Override
+ public boolean next(List result, int limit) throws IOException {
+ // We wont call this method at all.. As we have only CF:qualifier per row in index table.
+ throw new UnsupportedOperationException("Use only next(List results) method.");
+ }
+
+ @Override
+ public long getMaxResultSize() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public long getMvccReadPoint() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public boolean nextRaw(List result) throws IOException {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean nextRaw(List result, int limit) throws IOException {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NoIndexFilterNode.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NoIndexFilterNode.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NoIndexFilterNode.java (working copy)
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.index.Column;
+import org.apache.hadoop.hbase.index.IndexSpecification;
+import org.apache.hadoop.hbase.util.Pair;
+
+public class NoIndexFilterNode implements FilterNode {
+
+ @Override
+ public Map, IndexSpecification> getIndexToUse() {
+ return null;
+ }
+
+ @Override
+ public Map>> getPossibleUseIndices() {
+ return null;
+ }
+
+ @Override
+ public Map>> getPossibleFutureUseIndices() {
+ return null;
+ }
+}
\ No newline at end of file
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NonLeafFilterNode.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NonLeafFilterNode.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/NonLeafFilterNode.java (working copy)
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.index.Column;
+import org.apache.hadoop.hbase.index.GroupingCondition;
+import org.apache.hadoop.hbase.index.IndexSpecification;
+import org.apache.hadoop.hbase.util.Pair;
+
+public class NonLeafFilterNode implements FilterNode {
+
+ private List filterNodes = new ArrayList();
+
+ private GroupingCondition groupingCondition;
+
+ private Map, IndexSpecification> indicesToUse =
+ new HashMap, IndexSpecification>();
+
+ public NonLeafFilterNode(GroupingCondition condition) {
+ this.groupingCondition = condition;
+ }
+
+ public GroupingCondition getGroupingCondition() {
+ return groupingCondition;
+ }
+
+ public List getFilterNodes() {
+ return filterNodes;
+ }
+
+ public void addFilterNode(FilterNode filterNode) {
+ this.filterNodes.add(filterNode);
+ }
+
+ public void addIndicesToUse(FilterColumnValueDetail f, IndexSpecification i) {
+ List key = new ArrayList(1);
+ key.add(f);
+ this.indicesToUse.put(key, i);
+ }
+
+ public void addIndicesToUse(List lf, IndexSpecification i) {
+ this.indicesToUse.put(lf, i);
+ }
+
+ @Override
+ public Map, IndexSpecification> getIndexToUse() {
+ return this.indicesToUse;
+ }
+
+ @Override
+ public Map>> getPossibleUseIndices() {
+ return null;
+ }
+
+ @Override
+ public Map>> getPossibleFutureUseIndices() {
+ // There is no question of future use possible indices on a non leaf node.
+ return null;
+ }
+}
\ No newline at end of file
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/PossibleIndexFilterNode.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/PossibleIndexFilterNode.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/PossibleIndexFilterNode.java (working copy)
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.index.Column;
+import org.apache.hadoop.hbase.index.IndexSpecification;
+import org.apache.hadoop.hbase.util.Pair;
+
+public class PossibleIndexFilterNode implements LeafFilterNode {
+
+ private List> possibleFutureUseIndices;
+
+ private FilterColumnValueDetail filterColumnValueDetail;
+
+ public PossibleIndexFilterNode(List> possibleFutureUseIndices,
+ FilterColumnValueDetail filterColumnValueDetail) {
+ this.possibleFutureUseIndices = possibleFutureUseIndices;
+ this.filterColumnValueDetail = filterColumnValueDetail;
+ }
+
+ @Override
+ public Map>> getPossibleFutureUseIndices() {
+ // TODO avoid create of Map instance all the time...
+ Map>> reply =
+ new HashMap>>();
+ reply.put(filterColumnValueDetail.getColumn(), possibleFutureUseIndices);
+ return reply;
+ }
+
+ @Override
+ public Map, IndexSpecification> getIndexToUse() {
+ return null;
+ }
+
+ public Map>> getPossibleUseIndices() {
+ return null;
+ }
+
+ @Override
+ public FilterColumnValueDetail getFilterColumnValueDetail() {
+ return this.filterColumnValueDetail;
+ }
+
+ @Override
+ public void setFilterColumnValueDetail(FilterColumnValueDetail filterColumnValueDetail) {
+ this.filterColumnValueDetail = filterColumnValueDetail;
+ }
+
+ @Override
+ public IndexSpecification getBestIndex() {
+ return null;
+ }
+}
\ No newline at end of file
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScanner.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScanner.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScanner.java (working copy)
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+
+public interface ReInitializableRegionScanner extends SeekAndReadRegionScanner {
+
+ // TODO better name
+ void reInit(RegionScanner rs) throws IOException;
+
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScannerImpl.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScannerImpl.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ReInitializableRegionScannerImpl.java (working copy)
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+
+// TODO better name?
+public class ReInitializableRegionScannerImpl implements ReInitializableRegionScanner {
+
+ private RegionScanner delegator;
+
+ private int batch;
+
+ private byte[] lastSeekedRowKey;
+
+ private byte[] lastSeekAttemptedRowKey;
+
+ private static final Log LOG = LogFactory.getLog(ReInitializableRegionScannerImpl.class);
+
+ // Can this be queue?
+ // Criteria for the selection to be additional heap overhead wrt the object
+ // used cost in operation
+ private Set seekPoints;
+
+ private SeekPointFetcher seekPointFetcher;
+
+ private boolean closed = false;
+
+ public ReInitializableRegionScannerImpl(RegionScanner delegator, Scan scan,
+ SeekPointFetcher seekPointFetcher) {
+ this.delegator = delegator;
+ this.batch = scan.getBatch();
+ this.seekPoints = new TreeSet(Bytes.BYTES_COMPARATOR);
+ this.seekPointFetcher = seekPointFetcher;
+ }
+
+ @Override
+ public HRegionInfo getRegionInfo() {
+ return this.delegator.getRegionInfo();
+ }
+
+ @Override
+ public boolean isFilterDone() throws IOException {
+ return this.delegator.isFilterDone();
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ try {
+ this.delegator.close();
+ } finally {
+ this.seekPointFetcher.close();
+ }
+ closed = true;
+ }
+
+ public boolean isClosed() {
+ return closed;
+ }
+
+ @Override
+ public void addSeekPoints(List seekPoints) {
+ // well this add will do the sorting and remove duplicates. :)
+ for (byte[] seekPoint : seekPoints) {
+ this.seekPoints.add(seekPoint);
+ }
+ }
+
+ @Override
+ public boolean seekToNextPoint() throws IOException {
+ // At this class level if seek is called it must be forward seek.
+ // call reseek() directly with the next seek point
+ Iterator spIterator = this.seekPoints.iterator();
+ if (spIterator.hasNext()) {
+ this.lastSeekAttemptedRowKey = spIterator.next();
+ if (null != this.lastSeekedRowKey
+ && Bytes.BYTES_COMPARATOR.compare(this.lastSeekedRowKey, this.lastSeekAttemptedRowKey) > 0) {
+ throw new SeekUnderValueException();
+ }
+ spIterator.remove();
+ LOG.trace("Next seek point " + Bytes.toString(this.lastSeekAttemptedRowKey));
+ boolean reseekResult = closed ? false : this.reseek(this.lastSeekAttemptedRowKey);
+ if (!reseekResult) return false;
+ this.lastSeekedRowKey = this.lastSeekAttemptedRowKey;
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized boolean reseek(byte[] row) throws IOException {
+ return this.delegator.reseek(row);
+ }
+
+ @Override
+ public void reInit(RegionScanner rs) throws IOException {
+ this.delegator.close();
+ this.delegator = rs;
+ this.lastSeekedRowKey = null;
+ this.lastSeekAttemptedRowKey = null;
+ this.closed = false;
+ }
+
+ @Override
+ public byte[] getLatestSeekpoint() {
+ return this.lastSeekAttemptedRowKey;
+ }
+
+ @Override
+ public long getMvccReadPoint() {
+ return this.delegator.getMvccReadPoint();
+ }
+
+ @Override
+ public boolean nextRaw(List result) throws IOException {
+ return this.delegator.nextRaw(result);
+ }
+
+ @Override
+ public boolean nextRaw(List result, int limit) throws IOException {
+ return this.delegator.nextRaw(result, limit);
+ }
+
+ @Override
+ public boolean next(List results) throws IOException {
+ return next(results, this.batch);
+ }
+
+ @Override
+ public boolean next(List| result, int limit) throws IOException {
+ // Before every next call seek to the appropriate position.
+ if (closed) return false;
+ while (!seekToNextPoint()) {
+ List seekPoints = new ArrayList(1);
+ // TODO Do we need to fetch more seekpoints here?
+ if (!closed) this.seekPointFetcher.nextSeekPoints(seekPoints, 1);
+ if (seekPoints.isEmpty()) {
+ // nothing further to fetch from the index table.
+ return false;
+ }
+ this.seekPoints.addAll(seekPoints);
+ }
+ return closed ? false : this.delegator.next(result, limit);
+ }
+
+ @Override
+ public long getMaxResultSize() {
+ return this.delegator.getMaxResultSize();
+ }
+}
Index: hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ScanFilterEvaluator.java
===================================================================
--- hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ScanFilterEvaluator.java (revision 0)
+++ hbase-secondaryindex/src/main/java/org/apache/hadoop/hbase/index/coprocessor/regionserver/ScanFilterEvaluator.java (working copy)
@@ -0,0 +1,1188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.index.coprocessor.regionserver;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.index.Column;
+import org.apache.hadoop.hbase.index.ColumnQualifier;
+import org.apache.hadoop.hbase.index.GroupingCondition;
+import org.apache.hadoop.hbase.index.ValuePartition;
+import org.apache.hadoop.hbase.index.ColumnQualifier.ValueType;
+import org.apache.hadoop.hbase.index.Constants;
+import org.apache.hadoop.hbase.index.IndexSpecification;
+import org.apache.hadoop.hbase.index.client.EqualsExpression;
+import org.apache.hadoop.hbase.index.client.IndexExpression;
+import org.apache.hadoop.hbase.index.client.MultiIndexExpression;
+import org.apache.hadoop.hbase.index.client.NoIndexExpression;
+import org.apache.hadoop.hbase.index.client.RangeExpression;
+import org.apache.hadoop.hbase.index.client.SingleIndexExpression;
+import org.apache.hadoop.hbase.index.filter.SingleColumnRangeFilter;
+import org.apache.hadoop.hbase.index.filter.SingleColumnValuePartitionFilter;
+import org.apache.hadoop.hbase.index.manager.IndexManager;
+import org.apache.hadoop.hbase.index.util.ByteArrayBuilder;
+import org.apache.hadoop.hbase.index.util.IndexUtils;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+
+public class ScanFilterEvaluator {
+
+ private static final Log LOG = LogFactory.getLog(ScanFilterEvaluator.class);
+
+ private static final Set EMPTY_INDEX_SET = new HashSet(0);
+
+ // TODO we can make this class singleton? This cache will be shared by all regions.
+ // Then need to change the Map to include the table info also. A Map | | | | | | | | | | | | | | | | | | | | | | | |